From 7529b2565f4b9ca0316d1710921d5857c1e479dd Mon Sep 17 00:00:00 2001 From: NumesSanguisU Date: Sat, 17 Aug 2019 19:24:01 +0000 Subject: [PATCH] Example project on how to combine Docker with ZeroMQ for micro-services --- examples/docker/LICENSE | 29 ++++++ examples/docker/README.md | 129 +++++++++++++++++++++++++++ examples/docker/docker-compose.yml | 19 ++++ examples/docker/pub/Dockerfile | 18 ++++ examples/docker/pub/main.py | 44 +++++++++ examples/docker/pub/requirements.txt | 1 + examples/docker/requirements.txt | 1 + examples/docker/sub/Dockerfile | 19 ++++ examples/docker/sub/main.py | 37 ++++++++ examples/docker/sub/requirements.txt | 1 + 10 files changed, 298 insertions(+) create mode 100644 examples/docker/LICENSE create mode 100644 examples/docker/README.md create mode 100644 examples/docker/docker-compose.yml create mode 100644 examples/docker/pub/Dockerfile create mode 100644 examples/docker/pub/main.py create mode 100644 examples/docker/pub/requirements.txt create mode 100644 examples/docker/requirements.txt create mode 100644 examples/docker/sub/Dockerfile create mode 100644 examples/docker/sub/main.py create mode 100644 examples/docker/sub/requirements.txt diff --git a/examples/docker/LICENSE b/examples/docker/LICENSE new file mode 100644 index 000000000..bfdf56636 --- /dev/null +++ b/examples/docker/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) [year], [fullname] +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/examples/docker/README.md b/examples/docker/README.md new file mode 100644 index 000000000..5459af798 --- /dev/null +++ b/examples/docker/README.md @@ -0,0 +1,129 @@ +Original repository: https://github.com/NumesSanguis/pyzmq-docker + +# Docker & ZeroMQ +## Overview +Example project to demonstrate how you can turn Python scripts +into micro-services in Docker containers, which can communicate over ZeroMQ. +The examples here can be run as just Python-Python, Docker-Docker (`docker-compose`) or Docker-Python (`docker run`). + +Examples are using a Publisher-Subscriber pattern to communicate. +This means that the publisher micro-service just send messages out to a port, +without knowing who is listening and a subscriber micro-service receiving data, +without knowing where the data comes from. + +With ZeroMQ, only 1 micro-service can `socket.bind(url)` to 1 address. +However, you can have unlimited micro-services `socket.connect(url)` to an address. +This means that you can either have many-pub to 1-sub (examples in this Git repo) or 1-pub to many-sub on 1 ip:port combination. + + +## Install Docker +* [General Docker instructions](https://docs.docker.com/install/#supported-platforms) +* [Docker Toolbox for Windows 7/8/10 Home](https://docs.docker.com/toolbox/overview/) +* [Docker for Windows 10 Pro, Enterprise or Education](https://docs.docker.com/docker-for-windows/install/#what-to-know-before-you-install) +* Ubuntu: [Docker](https://docs.docker.com/install/linux/docker-ce/ubuntu/) and [docker-compose](https://docs.docker.com/compose/install/) and `sudo usermod -a -G docker $USER` + + +## 1. Python-Python +1. Open a terminal and navigate to folder `pyzmq-docker/sub` +2. Execute `python main.py` +3. Open a terminal and navigate to folder `pyzmq-docker/pub` +4. Execute `python main.py` +5. See subscriber receiving messages from the publisher! + +Notes: +* Steps 1-2, can be reversed with steps 3-4. +* Make sure you've installed PyZMQ in your Python installation (`conda install pyzmq` or `pip install pyzmq`) + + +## 2. Docker-Docker with docker-compose +1. Open a terminal and navigate to folder `pyzmq-docker` +2. Execute`docker-compose up --build` +3. See a Dockerized subscriber receiving messages from a Dockerized publisher! (That's really everything? 0.o) + +Notes: +* If you didn't make any changes to your Docker container, you can Execute `docker-compose up` without `--build` + to skip the build process. +* Advantages of `docker-compose`: + * You need only 1 `docker-compose.yml` to start multiple Docker micro-services + * It connects the `pub` micro-service to the `sub` micro-service with `tcp://sub:5550`. + Docker automatically turns `sub` into the IP of the subscriber micro-service. + + +## 3. Docker-Python with docker run +Notes: +* Make sure you've installed PyZMQ in your Python installation (`conda install pyzmq` or `pip install pyzmq`) + +### 3a. pub-Docker, sub-Python +1. Open a terminal and navigate to folder `pyzmq-docker/sub` +2. Execute `python main.py` +3. Open file `pub/Dockerfile` and change `"yo.ur.i.p"` to your machine IP (something similar to: `"192.168.99.1"`) +4. Open a terminal and navigate to folder `pyzmq-docker/pub` +5. Execute `docker build . -t foo/pub` +6. Execute `docker run -it foo/pub` +7. See that your subscriber receives messages from your Dockerized publisher. + +Notes: +* Step 5 can be skipped after the first time if no changes were made to the Docker/Python files. +* Steps 1-2 can be reversed with steps 3-6. + +### 3b. pub-Python, sub-Docker +1. Open a terminal and navigate to folder `pyzmq-docker/sub` +2. Execute `docker build . -t foo/sub` +3. Execute `docker run -p 5551:5551 -it foo/sub` (maps port of Docker container to localhost) +4. Open a terminal and navigate to folder `pyzmq-docker/pub` +5. Execute `python main.py` +6. See that your Dockerized subscriber receives messages from your publisher. + +Notes: +* Steps 1-3 can be reversed with steps 4-5. +* Add a name to a container by adding `--name foo-sub` to `docker run ` +* In case of container name already in use, remove that container with: `docker rm foo-sub` + + + +## Other +### Inspiration +Stackoverflow question: https://stackoverflow.com/questions/53802691/pyzmq-dockerized-pub-sub-sub-wont-receive-messages + +### Useful Docker commands + + sudo usermod -a -G docker $USER # add current user to group docker on Linux systems (Ubuntu) + + docker build . -t foo/sub # build docker image + docker run -it foo/sub # run build docker image and enter interactive mode + docker run -p 5551:5551 -it foo/sub # same as above with mapping Docker port to host + docker run -p 5551:5551 --name foo-sub -it foo/sub # same as above with naming container + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pyzmq-docker_sub_1 # get ip of container + docker rm foo-sub # remove container by name + + docker-compose up # run docker-compose.yml + docker-compose build / docker-compose up --build # rebuild images in docker-compose.yml + + docker image ls # show docker images + docker container ls # show docker containers + docker exec -it pyzmq-docker_pub_1 # enter bash in container + docker attach pyzmq-docker_sub_1 # get + + To detach the tty without exiting the shell, use the escape sequence Ctrl+p + Ctrl+q + + docker rm $(docker ps -a -q) # Delete all containers + docker rmi $(docker images -q) # Delete all images + + +### Debug docker-machine IP not found (probably not necessary) +Docker machine working check: +* Open a terminal and Execute command: `docker-machine ip` + * Should return a Docker machine IP (likely `192.168.99.100`) + * If not, see section "Debug" (e.g. `Error: No machine name(s) specified and no "default" machine exists`) + +Debug attempts: +* Execute the command `docker-machine ls`. +* If nothing shows up, we have to add a new machine with `docker-machine create default`. +* If that gives the error `Error with pre-create check: "VBoxManage not found. + Make sure VirtualBox is installed and VBoxManage is in the path"`, + see if `which virtualbox` and `which VBoxManage` return paths. + If not, you likely need to install VirtualBox. Else, see debug links. +* Debug links: + * https://github.com/docker/machine/issues/4590 + * Windows: https://stackoverflow.com/questions/39966083/docker-machine-no-machine-name-no-default-exists + * Install VirtualBox: https://stackoverflow.com/questions/45836296/error-with-pre-create-check-vboxmanage-not-found-make-sure-virtualbox-is-inst diff --git a/examples/docker/docker-compose.yml b/examples/docker/docker-compose.yml new file mode 100644 index 000000000..c54d6dc1d --- /dev/null +++ b/examples/docker/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" +services: + sub: + build: + context: ./sub # Docker context from folder of this file; needed to include requirement.txt + dockerfile: Dockerfile + ports: + - "5550:5550" # map container interal 5550 port to publicly accessible 5550 port + # stdin_open: true # same as docker -i (interactive) + tty: true # same as docker -t (tty); see if sub actually receives pub messages + command: ["python", "main.py", "--ip", "0.0.0.0"] # sub module binds, so no need for a specific IP + + pub: + build: + context: ./pub + dockerfile: Dockerfile + # stdin_open: true # same as docker -i (interactive) + tty: true # same as docker -t (tty); see if pub actually publishes messages to sub + command: ["python", "main.py", "--ip", "sub"] # pub module connects, therefore sub Docker IP needed diff --git a/examples/docker/pub/Dockerfile b/examples/docker/pub/Dockerfile new file mode 100644 index 000000000..413f21979 --- /dev/null +++ b/examples/docker/pub/Dockerfile @@ -0,0 +1,18 @@ +#pub +FROM python:3.7.1-slim + +MAINTAINER Stef van der Struijk + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gcc + +WORKDIR /app +COPY requirements.txt /app +RUN pip install -r requirements.txt + +COPY main.py /app/main.py + +# when using docker-compose, this command can be overwritten +# Change "yo.ur.i.p" to your machine IP (something similar to: "192.168.99.1") when using `docker run ` +CMD ["python", "main.py", "--ip", "yo.ur.i.p"] diff --git a/examples/docker/pub/main.py b/examples/docker/pub/main.py new file mode 100644 index 000000000..6f6e57be5 --- /dev/null +++ b/examples/docker/pub/main.py @@ -0,0 +1,44 @@ +# BSD 3-Clause License +# Stef van der Struijk + +import argparse +import zmq +import time + + +def publisher(ip="0.0.0.0", port=5551): + # ZMQ connection + url = "tcp://{}:{}".format(ip, port) + print("Going to connect to: {}".format(url)) + ctx = zmq.Context() + socket = ctx.socket(zmq.PUB) + socket.connect(url) # publisher connects to subscriber + print("Pub connected to: {}\nSending data...".format(url)) + + i = 0 + + while True: + topic = 'foo'.encode('ascii') + msg = 'test {}'.format(i).encode('ascii') + # publish data + socket.send_multipart([topic, msg]) # 'test'.format(i) + print("On topic {}, send data: {}".format(topic, msg)) + time.sleep(.5) + + i += 1 + + +if __name__ == "__main__": + # command line arguments + parser = argparse.ArgumentParser() + parser.add_argument("--ip", default=argparse.SUPPRESS, + help="IP of (Docker) machine") + parser.add_argument("--port", default=argparse.SUPPRESS, + help="Port of (Docker) machine") + + args, leftovers = parser.parse_known_args() + print("The following arguments are used: {}".format(args)) + print("The following arguments are ignored: {}\n".format(leftovers)) + + # call function and pass on command line arguments + publisher(**vars(args)) diff --git a/examples/docker/pub/requirements.txt b/examples/docker/pub/requirements.txt new file mode 100644 index 000000000..02ec117e1 --- /dev/null +++ b/examples/docker/pub/requirements.txt @@ -0,0 +1 @@ +pyzmq diff --git a/examples/docker/requirements.txt b/examples/docker/requirements.txt new file mode 100644 index 000000000..02ec117e1 --- /dev/null +++ b/examples/docker/requirements.txt @@ -0,0 +1 @@ +pyzmq diff --git a/examples/docker/sub/Dockerfile b/examples/docker/sub/Dockerfile new file mode 100644 index 000000000..d22cfba56 --- /dev/null +++ b/examples/docker/sub/Dockerfile @@ -0,0 +1,19 @@ +#sub +FROM python:3.7.1-slim + +MAINTAINER Stef van der Struijk + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gcc + +WORKDIR /app +COPY requirements.txt /app +RUN pip install -r requirements.txt +COPY main.py /app/main.py + +# allow other containers/PCs to connect; maybe not necessary +EXPOSE 5551 + +# when using docker-compose, this command can be overwritten +CMD ["python", "main.py", "--ip", "0.0.0.0"] diff --git a/examples/docker/sub/main.py b/examples/docker/sub/main.py new file mode 100644 index 000000000..51892ed6a --- /dev/null +++ b/examples/docker/sub/main.py @@ -0,0 +1,37 @@ +# BSD 3-Clause License +# Stef van der Struijk + +import argparse +import zmq + + +def subscriber(ip="0.0.0.0", port=5551): + # ZMQ connection + url = "tcp://{}:{}".format(ip, port) + print("Going to bind to: {}".format(url)) + ctx = zmq.Context() + socket = ctx.socket(zmq.SUB) + socket.bind(url) # subscriber creates ZeroMQ socket + socket.setsockopt(zmq.SUBSCRIBE, ''.encode('ascii')) # any topic + print("Sub bound to: {}\nWaiting for data...".format(url)) + + while True: + # wait for publisher data + topic, msg = socket.recv_multipart() + print("On topic {}, received data: {}".format(topic, msg)) + + +if __name__ == "__main__": + # command line arguments + parser = argparse.ArgumentParser() + parser.add_argument("--ip", default=argparse.SUPPRESS, + help="IP of (Docker) machine") + parser.add_argument("--port", default=argparse.SUPPRESS, + help="Port of (Docker) machine") + + args, leftovers = parser.parse_known_args() + print("The following arguments are used: {}".format(args)) + print("The following arguments are ignored: {}\n".format(leftovers)) + + # call function and pass on command line arguments + subscriber(**vars(args)) diff --git a/examples/docker/sub/requirements.txt b/examples/docker/sub/requirements.txt new file mode 100644 index 000000000..02ec117e1 --- /dev/null +++ b/examples/docker/sub/requirements.txt @@ -0,0 +1 @@ +pyzmq