properly starting olregistry and rethinkdb instances

edoakes · edoakes · commit eeeda4193a39 · 2016-08-24T19:19:45.000-04:00
diff --git a/util/cluster_manager.py b/util/cluster_manager.py
@@ -54,33 +54,50 @@ def write_reg(self, cid):
         config = {'cid': cid,
                   'ip': self.registry_ip,
                   'host_ip': self.host_ip,
-                  'host_port': self.registry_port,
-                  'type': 'registry'}
+                  'host_port': self.registry_port, 'type': 'registry'}
         wrjs(config_path, config)
 
         print 'Started registry "localhost:%s"' % self.registry_port
 
-    def start_olreg(self):
+    def start_olreg(self, num):
+        nodes = []
         c = 'docker run -d -p 28015:28015 -p 29015:29015 rethinkdb rethinkdb --bind all'
-
-        # for subsequent hosts in cluster:
-        #c = 'docker run -d -p 28015:28015 -p 29015:29015 dockerfile/rethinkdb rethinkdb --bind all -j <first-host-ip>:29015'
         cid = run(c).strip()
-
-        self.rethinkdb_ip = container_ip(cid)
-        self.rethinkdb_port = lookup_host_port(cid, 28015)
-
-        print 'Started rethinkdb container "%s:%s"' % (self.rethinkdb_ip, "28015")
-
-        config_path = os.path.join(self.cluster_dir, 'registry.json')
-        config = {'cid': cid,
-                  'ip': self.rethinkdb_ip,
-                  'host_ip': self.host_ip,
-                  'host_port': self.rethinkdb_port,
-                  'type': 'rethinkdb'}
+        cluster_ip = container_ip(cid)
+        cluster_port = lookup_host_port(cid, 28015)
+        nodes.append({'cid': cid,
+                    'ip': cluster_ip,
+                    'host_ip': self.host_ip,
+                    'host_port': cluster_port})
+
+        print 'Started rethinkdb instance "localhost:28015"'
+        print 40*'='
+
+        for k in range(1, num):
+            client_port = 28015 + k
+            comm_port = 29015 + k
+            c = 'docker run -d -p %s:%s -p %s:%s rethinkdb rethinkdb --port-offset %s --bind all -j %s:29015' % (client_port, client_port, comm_port, comm_port, k, cluster_ip)
+            cid = run(c).strip()
+            host_port = lookup_host_port(cid, 28015+k)
+            nodes.append({'cid': cid,
+                        'ip': container_ip(cid),
+                        'host_ip': self.host_ip,
+                        'host_port': host_port})
+
+            print 'Started rethinkdb container "localhost:%s"' % host_port
+            print 40*'=' 
+
+        config_path = os.path.join(self.cluster_dir, 'rethinkdb.json')
+        config = {
+            'cluster': nodes,
+            'ip': nodes[0]['ip'],
+            'host_ip': self.host_ip,
+            'host_port': nodes[0]['host_port'],
+            'type': 'rethinkdb'
+        }
         wrjs(config_path, config)
 
-        c = 'docker run -d -p 0:%s olregistry /open-lambda/registry %s:%s' % (self.internal_reg_port, self.rethinkdb_ip, "28015")
+        c = 'docker run -d -p 0:%s olregistry /open-lambda/registry %s:%s' % (self.internal_reg_port, cluster_ip, cluster_port)
         cid = run(c).strip()
 
         self.write_reg(cid)
@@ -115,7 +132,7 @@ def start_workers(self):
             config['host_port'] = lookup_host_port(cid, self.internal_worker_port)
             wrjs(config_path, config, atomic=True)
 
-            print 'Started worker "localhost:%s"' % config['host_port']
+            print 'Started worker container "localhost:%s"' % config['host_port']
             self.workers.append(config)
 
     def start_lb(self):
@@ -137,7 +154,7 @@ def start_lb(self):
         config_path = os.path.join(self.cluster_dir, 'loadbalancer-%d.json' % 1)
         wrjs(config_path, config, atomic=True)
 
-        print 'Started loadbalancer "localhost:%s"' % self.balancer_port
+        print 'Started loadbalancer container "localhost:%s"' % self.balancer_port
 
     def write_nginx_config(self, path):
         config = 'http {\n\tupstream handlers {\n'
@@ -158,7 +175,7 @@ def rethinkdb_wait(self):
                 if up < len(self.workers):
                     print '%d of %d rethinkdb instances are ready' % (up, len(self.workers))
             except:
-                print 'waiting for first rethinkdb instance to come up'
+                print 'Waiting for first rethinkdb instance to come up'
 
             time.sleep(1)
 
diff --git a/util/start-cluster.py b/util/start-cluster.py
@@ -14,6 +14,7 @@ def main():
     parser.add_argument('--registry', '-r', default='docker')
     parser.add_argument(DB_WAIT, default=False, action='store_true')
     parser.add_argument('--remote', default=False, action='store_true')
+    parser.add_argument('--nrethink', default='1')
 
     global args
     args = parser.parse_args()
@@ -33,7 +34,7 @@ def main():
     print '='*40
 
     if args.registry == "olregistry":
-        cluster.start_olreg()
+        cluster.start_olreg(int(args.nrethink))
     else:
         cluster.start_docker_reg()
     print '='*40
diff --git a/util/stop-cluster.py b/util/stop-cluster.py
@@ -13,7 +13,6 @@ def main():
     cluster_dir = os.path.join(SCRIPT_DIR, args.cluster)
     if not os.path.exists(cluster_dir):
         if args.if_running:
-            # it's OK, we weren't assuming it was running
             sys.exit(0)
         else:
             print 'Cluster not running!'
@@ -25,6 +24,18 @@ def main():
             continue
         try:
             info = rdjs(path)
+            if info['type'] == 'rethinkdb':
+                for node in info['cluster']:
+                    try:
+                        run('docker kill '+node['cid'])
+                    except Exception as e:
+                        if args.force:
+                            print 'continue because force was used (cleanup may not be complete)'
+                        else:
+                            raise e
+                os.remove(path)
+                print 'killed ' + info['ip'] + ' (' + filename + ')'
+                continue
             cid = info['cid']
             if info['type'] == 'worker':
                 # need this script, otherwise it hangs if Docker inside
