Snippet:Uploading the model and running cplex solver on the cluster

From HUES Platform Wiki
Description Calls AIMMS, generates .mps and .lis file, creates all other necessary files for the cluster, connects to the cluster using SSH and uploads the needed files, runs the job, delets original files on the hard drive.

Host, username, password, paths and settings should be changed for the specific cluster.

Author(s) Boran Morvaj
Language Python
Tags python, cplex, aimms, milp
Snippet code ########SCRIPT#########
import os
import subprocess 
import paramiko
class SSHConnection(object):
""""""

 #----------------------------------------------------------------------
 def __init__(self, host, username, password, port=22):
  """Initialize and setup connection"""
  self.sftp = None
  self.sftp_open = False
  self.s = None
  self.password = password
  self.username= username
  self.host= host
  self.port=port

  # open SSH Transport stream
  self.transport = paramiko.Transport((host, port))

  self.transport.connect(username=username, password=password)
 
 
 
 #----------------------------------------------------------------------
 def _openSSHClient(self):
  """
  Opens an SFTP connection if not already open
  """
  self.s = paramiko.SSHClient()
  self.s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  self.s.connect(self.host, self.port, self.username, self.password)
 

 #----------------------------------------------------------------------
 def _openSFTPConnection(self):
  """
  Opens an SFTP connection if not already open
  """
  if not self.sftp_open:
   self.sftp = paramiko.SFTPClient.from_transport(self.transport)
   self.sftp_open = True
  

 #----------------------------------------------------------------------
 def get(self, remote_path, local_path=None):
  """
  Copies a file from the remote host to the local host.
  """
  self._openSFTPConnection()        
  self.sftp.get(remote_path, local_path)        

 #----------------------------------------------------------------------
 def put(self, local_path, remote_path=None):
  """
  Copies a file from the local host to the remote host
  """
  self._openSFTPConnection()
  self.sftp.put(local_path, remote_path)

 #----------------------------------------------------------------------
 def do_it(self, command):
 
  
  Run the command
  
  self._openSSHClient()        
 
  (stdin, stdout, stderr) = self.s.exec_command(command)
  for line in stdout.readlines():
   print line
 
  self.s.close()
 
 #----------------------------------------------------------------------
 def close(self):
  """
  Close SFTP connection and ssh connection
  """
  if self.sftp_open:
   self.sftp.close()
   self.sftp_open = False
  
host = "hostname.ch"
username = "username"
pw = "password"


############AIMMS#############

modelname = 'DG'
projectpath=os.path.join(r'C:\Users\...\DG.aimms')

subprocess.check_call(["C:\Program Files\AIMMS\AIMMS 4\Bin\Aimms.exe", "-m", "--run-only", "MainExecution",  projectpath])


##pbs_script.txt
pbs=open(r'C:\Users\...\pbs_script.txt','w')
pbsline1="# Job name\n"
pbsline2="#PBS -N normnnoERnoPWL\n"
pbsline3="# Job will run for maximum of 30 min\n"
pbsline4="#PBS -l walltime=300:00:00\n"
pbsline5="# 1 nodes and 16 cores per node\n"
pbsline6="#PBS -l nodes=1:ppn=8\n"
pbsline7="# Request email at the start and end of the job\n"
pbsline8="#PBS -m abe\n"
pbsline9="# Email address to use\n"
pbsline10="#PBS -M boran.morvaj@empa.ch\n"
pbsline11="# Set the working directory\n"
pbsline12="#PBS -d /mnt/test\n"
pbsline13="# Select a queue (see qstat)\n"
pbsline14="#PBS -q infinijazz\n"
pbsline15="cplex < cplex_script.txt"

pbs.writelines([pbsline1, pbsline2, pbsline3, pbsline4, pbsline5, pbsline6, pbsline7, pbsline8, pbsline9, pbsline10, 
    pbsline11, pbsline12, pbsline13, pbsline14, pbsline15])

pbs.close()

#cplex

cplex=open(r'C:\Users\mobo\...\cplex_script.txt','w')
cplexline1="read /mnt/project/case.mps\n"
cplexline2="read /mnt/project/cplex_settings.prm\n"
cplexline3="optimize\n"
cplexline4="write /mnt/project/result.mst\n"
cplexline5="write /mnt/project/result.sol\n"
cplexline6="quit"

cplex.writelines([cplexline1, cplexline2, cplexline3, cplexline4, cplexline5, cplexline6])

cplex.close()

#cplex settings

settings=open(r'C:\Users\mobo\...\cplex_settings.prm','w')

settingsline1="CPLEX Parameter File Version 12.6\n"
settingsline2="CPX_PARAM_EPGAP 0.05\n"
settingsline3="CPX_PARAM_THREADS 8\n"
settingsline4="CPX_PARAM_TILIM 1070000\n"
settingsline5="CPX_PARAM_WORKMEM 23500\n"
settingsline6="CPX_PARAM_NODEFILEIND 3\n"
settingsline7="CPX_PARAM_PARALLELMODE -1\n"
settingsline8="CPX_PARAM_MIPDISPLAY 4\n"
settingsline9="CPX_PARAM_MIPEMPHASIS 0\n"
settingsline10="CPX_PARAM_EPRHS 1e-05\n"

settings.writelines([settingsline1, settingsline2, settingsline3, settingsline4, settingsline5, settingsline6, settingsline7,
      settingsline8, settingsline9, settingsline10])

settings.close()

origin1 = r'C:\Users\...\pbs_script.txt'
dst1 = '/mnt/project/pbs_script.txt'

origin2 = r'C:\Users\...\cplex_script.txt'
dst2 = '/mnt/project/cplex_script.txt'

origin3 = r'C:\Users\...cplex_settings.prm'
dst3 = '/mnt/project/cplex_settings.prm'

origin4 = r'C:\Users\...cpx00000.mps'
dst4 = '/mnt/project/test.mps'

origin5 = r'C:\Users\...DG.lis'
dst5 = '/mnt/project/test.lis'


makedir='mkdir -p /mnt/project'
run='cd /mnt/project' +' \n qsub pbs_script.txt'

ssh = SSHConnection(host, username, pw)
ssh.do_it(makedir)
ssh.put(origin1, dst1)
ssh.put(origin2, dst2)
ssh.put(origin3, dst3)
ssh.put(origin4, dst4)
ssh.put(origin5, dst5)
ssh.do_it(run)
ssh.close()

###delete mps and lis ###

os.remove(r'C:\Users\...\cpx00000.mps')
os.remove(r'C:\Users\...\DG.lis')
self.transport.close()
Notes