Merge pull request #200 from alphaville/feature/198

TCP server with dynamically configurable ip and port

Author: alphaville

.travis.yml

@@ -75,6 +75,9 @@ before_install:
 install:
   - bash ci/install.sh
 
+before_script:
+  - rustup component add clippy
+
 script:
   - bash ci/script.sh
 

ci/script.sh

@@ -6,6 +6,9 @@ regular_test() {
     # ------------------------------------
     cargo test
 
+    # Run Clippy
+    # ------------------------------------
+    cargo clippy --all-targets --all-features
 
     # Run Python tests
     # ------------------------------------
@@ -29,6 +32,16 @@ regular_test() {
     export PYTHONPATH=.
     python -W ignore test/test_constraints.py -v
     python -W ignore test/test.py -v
+
+
+    # Run Clippy for generated optimizers
+    # ------------------------------------
+    cd .python_test_build/only_f1/tcp_iface_only_f1
+    cargo clippy --all-targets --all-features
+    cd ../../only_f2/tcp_iface_only_f2
+    cargo clippy --all-targets --all-features
+    cd ../../rosenbrock_ros
+    cargo clippy --all-targets --all-features
 }
 
 test_docker() {

docs/python-advanced.md

@@ -126,7 +126,7 @@ for requests (e.g., for remote connections), you may crate an
 ```python
 tcp_config = og.config.TcpServerConfiguration('10.8.0.12', 9555)
 ```
-
+                               
 and then provide it to the builder configuration using 
 
 ```python
@@ -138,10 +138,13 @@ builder_config.with_tcp_interface_config(tcp_config)
 There are two ways to connect to a generated TCP server and call the
 auto-generated optimizer:
 
-1. *Connect to a local optimizer* by providing the path of the optimizer
+<div class="alert alert-info">
+<b>Connect to a local optimizer</b> by providing the path of the optimizer
 directory. For that purpose, we need to create an instance of 
-`OptimizerTcpManager` and specify the path to the auto-generated optimizer;
-for example,
+<code>OptimizerTcpManager</code> and specify the path to the auto-generated optimizer.
+</div>
+
+For example,
 
 ```python
 mng = og.tcp.OptimizerTcpManager('python_build/the_optimizer')
@@ -150,10 +153,28 @@ mng = og.tcp.OptimizerTcpManager('python_build/the_optimizer')
 we can then `start` the optimizer. The TCP manager known what IP and port 
 to link to, so we can `call` it directly.
 
-2. *Connect to a remote optimizer* by providing its IP and port. In that 
+<div class="alert alert-info">
+<b>Connect to a local optimizer</b> and <b>customize</b> its IP and port. 
+This is particularly useful if you need to start multiple instances of a TCP 
+server (with different ports).
+</div>
+
+For example,
+
+```python
+ip = '0.0.0.0'
+port = 5678
+mng = og.tcp.OptimizerTcpManager('python_build/the_optimizer', ip, port)
+```
+
+<div class="alert alert-info">
+<b>Connect to a remote optimizer</b> by providing its IP and port. In that 
 case we assume that an optimizer is up an running at some remote address
 and listens for connections at a certain port. In that case, we cannot 
-`start` the optimizer remotely using the TCP manager. 
+<code>start</code> the optimizer remotely using the TCP manager. 
+</div>
+
+
 
 For example to connect to a *remote* TCP server at `10.8.0.7:5678`, we can
 create a TCP manager as follows:

open-codegen/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 Note: This is the Changelog file of `opengen` - the Python interface of OpEn
 
+## [0.6.1] - 2020-09-10
+
+### Changed
+
+* `OptimizerTcpManager`: ip and port can be set dynamically
+
+
 ## [0.6.0] - 2020-09-03
 
 ### Added 
@@ -55,6 +62,7 @@ Note: This is the Changelog file of `opengen` - the Python interface of OpEn
 * Fixed `lbfgs` typo
 
 
+[0.6.1]: https://github.com/alphaville/optimization-engine/compare/opengen-v0.6.0...opengen-0.6.1
 [0.6.0]: https://github.com/alphaville/optimization-engine/compare/opengen-v0.5.0...opengen-0.6.0
 [0.5.0]: https://github.com/alphaville/optimization-engine/compare/opengen-0.4.1...opengen-v0.5.0
 [0.4.1]: https://github.com/alphaville/optimization-engine/compare/opengen-0.4.1...master

open-codegen/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

open-codegen/opengen/main.py

@@ -1,6 +1,10 @@
+import time
+import logging
 import casadi.casadi as cs
 import opengen as og
 
+logging.getLogger().setLevel(5)
+
 u = cs.SX.sym("u", 5)  # decision variable (nu = 5)
 p = cs.SX.sym("p", 2)  # parameter (np = 2)
 phi = cs.dot(u, u)  # cost function
@@ -11,18 +15,39 @@
     .with_constraints(bounds)
 
 meta = og.config.OptimizerMeta() \
-    .with_optimizer_name("halfspace_optimizer")
+    .with_optimizer_name("halfspace_optimizer") \
+    .with_authors(["P. Sopasakis", "S. Author"]).with_version("0.1.56")
 
 tcp_config = og.config.TcpServerConfiguration(bind_port=3305)
 build_config = og.config.BuildConfiguration() \
     .with_build_directory('my_optimizers') \
     .with_build_mode(og.config.BuildConfiguration.DEBUG_MODE) \
-    .with_open_version('0.7.1-alpha') \
-    .with_tcp_interface_config(tcp_interface_config=tcp_config)
+    .with_tcp_interface_config()
 
 builder = og.builder.OpEnOptimizerBuilder(problem,
                                           meta,
                                           build_config,
                                           og.config.SolverConfiguration())
 builder.build()
 
+all_managers = []
+for i in range(10):
+    all_managers += [og.tcp.OptimizerTcpManager(
+        optimizer_path='my_optimizers/halfspace_optimizer',
+        ip='0.0.0.0',
+        port=3311+i)]
+
+for m in all_managers:
+    m.start()
+
+time.sleep(4)
+
+for m in all_managers:
+    print(m.details)
+    resp = m.call(p=[1., 2.])
+    print(resp.get().solution)
+
+# mng.kill()
+time.sleep(6)
+for m in all_managers:
+    m.kill()

open-codegen/opengen/tcp/optimizer_tcp_manager.py

@@ -20,54 +20,70 @@ class OptimizerTcpManager:
     """
 
     def __init__(self, optimizer_path=None, ip=None, port=None):
-        """Constructs instance of <code>OptimizerTcpManager</code>
+        """
+        Constructs instance of <code>OptimizerTcpManager</code>
+
+        There are three ways to use this constructor:
+
+        - OptimizerTcpManager(optimizer_path): creates a TCP manager for a local
+          TCP server using the default IP and port of that TCP server (specified
+          upon code generation)
+        - OptimizerTcpManager(optimizer_path, ip, port): creates a TCP manager
+          for a local TCP server, but overrides the default IP and port. This way
+          the user can set the address '0.0.0.0', so that the TCP server binds on
+          all IPs, or '127.0.0.1' so that it is accessible only locally, or a VPN
+          IP address, so that the optimizer is accessible only over a private
+          network.
+        - OptimizerTcpManager(ip, port): If a path is not provided, then the
+          TCP manager can be used to connect to a remote TCP server, as a client,
+          but cannot be used to start the server.
 
         Args:
-            optimizer_path: path to auto-generated optimizer (just to
-            be clear: this is the folder that contains <code>optimizer.yml</code>)
+            :param optimizer_path:
+            path to auto-generated optimizer (just to be clear: this is
+            the folder that contains <code>optimizer.yml</code>)
+
+            :param ip:
+            the user can provide the IP of a remote TCP server (must be up and
+            running) so as to establish a remote connection. In that case `path`
+            must be equal to `None` (see examples above)
+
+            :param port: see ip
 
         Returns:
             New instance of <code>OptimizerTcpManager</code>
         """
         self.__optimizer_path = optimizer_path
         if optimizer_path is not None:
-            self.__optimizer_details_from_yml = None
+            self.__optimizer_details = None  # create attribute (including IP and port)
             self.__load_tcp_details()
+            if ip is not None:
+                self.__optimizer_details['tcp']['ip'] = ip
+            if port is not None:
+                self.__optimizer_details['tcp']['port'] = port
         elif ip is not None and port is not None:
-            self.__optimizer_details_from_yml = {"tcp": {"ip": ip, "port": port}}
+            self.__optimizer_details = {"tcp": {"ip": ip, "port": port}}
         else:
             raise Exception("Illegal arguments")
         # Check whether the optimizer was built with the current version of opengen
-        opengen_version = self.__optimizer_details_from_yml['build']['opengen_version']
+        opengen_version = self.__optimizer_details['build']['opengen_version']
         current_opengen_version = pkg_resources.require("opengen")[0].version
         if current_opengen_version != opengen_version:
             logging.warn('the target optimizer was build with a different version of opengen (%s)' % opengen_version)
             logging.warn('you are running opengen version %s' % current_opengen_version)
 
+        logging.info("TCP/IP details: %s:%d",
+                     self.__optimizer_details['tcp']['ip'],
+                     self.__optimizer_details['tcp']['port'])
+
     def __load_tcp_details(self):
-        logging.info("loading TCP/IP details")
         yaml_file = os.path.join(self.__optimizer_path, "optimizer.yml")
         with open(yaml_file, 'r') as stream:
-            self.__optimizer_details_from_yml = yaml.safe_load(stream)
-        details = self.__optimizer_details_from_yml
-        logging.info("TCP/IP details: %s:%d", details['tcp']['ip'], details['tcp']['port'])
-
-    def __threaded_start(self):
-        optimizer_details = self.__optimizer_details_from_yml
-        logging.info("Starting TCP/IP server at %s:%d (in a detached thread)",
-                     optimizer_details['tcp']['ip'],
-                     optimizer_details['tcp']['port'])
-        command = ['cargo', 'run', '-q']
-        if optimizer_details['build']['build_mode'] == 'release':
-            command.append('--release')
-        tcp_dir_name = "tcp_iface_" + optimizer_details['meta']['optimizer_name']
-        tcp_iface_directory = os.path.join(self.__optimizer_path, tcp_dir_name)
-        p = subprocess.Popen(command, cwd=tcp_iface_directory)
-        p.wait()
+            self.__optimizer_details = yaml.safe_load(stream)
 
     @retry(tries=10, delay=1)
     def __obtain_socket_connection(self):
-        tcp_data = self.__optimizer_details_from_yml
+        tcp_data = self.__optimizer_details
         ip = tcp_data['tcp']['ip']
         port = tcp_data['tcp']['port']
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
@@ -101,31 +117,62 @@ def ping(self):
         return json.loads(data)
 
     def __check_if_server_is_running(self):
-        tcp_data = self.__optimizer_details_from_yml
+        tcp_data = self.__optimizer_details
         ip = tcp_data['tcp']['ip']
         port = tcp_data['tcp']['port']
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
         return 0 == s.connect_ex((ip, port))
 
+    @property
+    def details(self):
+        return self.__optimizer_details
+
     def start(self):
-        """Starts the TCP server"""
-        # start the server in a separate thread
+        """Starts the TCP server
+
+        Note: this method starts a *local* server whose path must have been
+        provided - we cannot start a remote server.
+
+        The server starts on a separate thread, so this method does not block
+        the execution of the caller's programme.
 
+        """
+
+        # Check if a path has been provided; if not,
         if self.__optimizer_path is None:
             raise Exception("No optimizer path provided - cannot start a remote server")
 
+        # Server start data
+        tcp_data = self.__optimizer_details
+        ip = tcp_data['tcp']['ip']
+        port = tcp_data['tcp']['port']
+
+        # Check if any of the ip/port pairs is occupied
         if self.__check_if_server_is_running():
-            msg = "Port %d not available" % self.__optimizer_details_from_yml['tcp']['port']
+            msg = "Port %d not available" % port
             raise Exception(msg)
 
+        def threaded_start():
+            optimizer_details = self.__optimizer_details
+            logging.info("Starting TCP/IP server at %s:%d (in a detached thread)",
+                         ip, port)
+            command = ['cargo', 'run', '-q', '--', '--port=%d' % port, '--ip=%s' % ip]
+            if optimizer_details['build']['build_mode'] == 'release':
+                command.append('--release')
+            tcp_dir_name = "tcp_iface_" + optimizer_details['meta']['optimizer_name']
+            tcp_iface_directory = os.path.join(self.__optimizer_path, tcp_dir_name)
+            p = subprocess.Popen(command, cwd=tcp_iface_directory)
+            p.wait()
+
+        # start the server in a separate thread
         logging.info("Starting TCP/IP server thread")
-        thread = Thread(target=self.__threaded_start)
+        thread = Thread(target=threaded_start)
         thread.start()
 
         # ping the server until it responds so that we know it's
         # up and running
         logging.info("Waiting for server to start")
-        time.sleep(2)
+        time.sleep(0.1)
         self.ping()
 
     def kill(self):