Add support for deploying via dockerflow pipeline. (#90) r=vladikoff

Dockerfile

@@ -1,48 +1,31 @@
-##########################################################
+FROM python:2.7-slim
-#                  /!\ WARNING /!\                       #
-# This is completely experimental. Use at your own risk. #
-#             Also, learn you some docker:               #
-#           http://docker.io/gettingstarted              #
-##########################################################
 
-FROM debian:7.4
+RUN groupadd --gid 1001 app && \
-MAINTAINER Dan Callahan <dan.callahan@gmail.com>
+    useradd --uid 1001 --gid 1001 --shell /usr/sbin/nologin app
-
-# Base system setup
-
-RUN DEBIAN_FRONTEND=noninteractive apt-get update \
-    && apt-get install --no-install-recommends -y \
-    vim locales \
-    && apt-get clean
-
-RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale
 
 ENV LANG C.UTF-8
 
-RUN useradd --create-home app
+WORKDIR /app
 
-# Build the Sync server
+# S3 bucket in Cloud Services prod IAM
+ADD https://s3.amazonaws.com/dumb-init-dist/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init
+RUN chmod +x /usr/local/bin/dumb-init
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+# install syncserver dependencies
-    ca-certificates \
+COPY ./requirements.txt /app/requirements.txt
-    build-essential \
+COPY ./dev-requirements.txt /app/dev-requirements.txt
-    libzmq-dev \
+RUN apt-get -q update \
-    python-dev \
+    && apt-get -q --yes install g++ \
-    python-virtualenv \
+    && pip install --upgrade --no-cache-dir -r requirements.txt \
+    && pip install --upgrade --no-cache-dir -r dev-requirements.txt \
+    && apt-get -q --yes remove g++ \
+    && apt-get -q --yes autoremove \
     && apt-get clean
 
+COPY ./syncserver /app/syncserver
+COPY ./setup.py /app
+RUN python ./setup.py develop
+
+# run as non priviledged user
 USER app
-
-RUN mkdir -p /home/app/syncserver
-ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/
-ADD ./syncserver/ /home/app/syncserver/syncserver/
-WORKDIR /home/app/syncserver
-
-RUN make build
-
-# Run the Sync server
-
-EXPOSE 5000
-
-ENTRYPOINT ["/usr/bin/make"]
-CMD ["serve"]

Makefile

@@ -31,7 +31,7 @@ test: | $(TOOLS)
 	# Tokenserver tests currently broken due to incorrect file paths
 	# $(ENV)/bin/nosetests -s tokenserver.tests
 	
-	# Test against a running server
+	# Test against a running server.
 	$(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \
 	sleep 2; \
 	$(ENV)/bin/python -m syncstorage.tests.functional.test_storage \
@@ -39,7 +39,7 @@ test: | $(TOOLS)
 	kill $$SERVER_PID
 
 $(TOOLS): | $(ENV)/COMPLETE
-	$(INSTALL) nose flake8
+	$(INSTALL) -r dev-requirements.txt
 
 .PHONY: serve
 serve: | $(ENV)/COMPLETE

README.rst

@@ -3,7 +3,7 @@ Run-Your-Own Firefox Sync Server
 
 This is an all-in-one package for running a self-hosted Firefox Sync server.
 It bundles the "tokenserver" project for authentication and the "syncstorage"
-project for storage, produce a single stand-alone webapp.
+project for storage, to produce a single stand-alone webapp.
 
 Complete installation instructions are available at:
 
@@ -13,7 +13,7 @@ Complete installation instructions are available at:
 Quickstart
 ----------
 
-The Sync Server software runs using **python 2.6** or later, and the build
+The Sync Server software runs using **python 2.7**, and the build
 process requires **make** and **virtualenv**.  You will need to have the
 following packages (or similar, depending on your operating system) installed:
 
@@ -87,6 +87,40 @@ to install an appropriate python module, e.g::
     $ ./local/bin/pip install psycopg2
 
 
+Runner under Docker
+-------------------
+
+There is experimental support for running the server inside a Docker
+container.  Build the image like this::
+
+    $ docker build -t syncserver:latest .
+
+Then you can run the server by passing in configuration options as
+environmet variables, like this::
+
+    $ docker run --rm \
+        # Expose the port that the server will listen on \
+        --network host \
+        -p 5000:5000 \
+        # Set important config options through environment variables
+        -e SYNCSERVER_PUBLIC_URL=http://localhost:5000 \
+        -e SYNCSERVER_SECRET=5up3rS3kr1t \
+        -e SYNCSERVER_SQLURI=sqlite:////tmp/syncserver.db \
+        -e SYNCSERVER_BATCH_UPLOAD_ENABLED=true \
+        # Run the container we just build \
+        syncserver:latest \
+        # Start gunicorn on the desired localhost port \
+        /usr/local/bin/gunicorn --bind localhost:5000 \
+        # And have it run the syncserver application \
+        syncserver.wsgi_app
+
+And you can test whether it's running correctly by using the builtin
+function test suite, like so::
+
+    $ /local/bin/python -m syncstorage.tests.functional.test_storage \
+        --use-token-server http://localhost:5000/token/1.0/sync/1.5
+
+
 
 Questions, Feedback
 -------------------

circle.yml

@@ -0,0 +1,68 @@
+# These environment variables must be set in CircleCI UI
+#
+# DOCKERHUB_REPO - docker hub repo, format: <username>/<repo>
+# DOCKER_EMAIL   - login info for docker hub
+# DOCKER_USER
+# DOCKER_PASS
+#
+machine:
+  services:
+    - docker
+
+dependencies:
+  # make sure to keep the docker cache dir
+  cache_directories:
+    - "~/docker"
+
+  override:
+    - docker info
+
+    # build the container, use circleci's docker cache workaround
+    # only use 1 image per day to keep the cache size from getting 
+    # too big and slowing down the build
+    - I="image-$(date +%j).tar"; if [[ -e ~/docker/$I ]]; then echo "Loading $I"; docker load -i ~/docker/$I; fi
+
+    # create version.json
+    - >
+        printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' 
+        "$CIRCLE_SHA1" 
+        "$CIRCLE_TAG" 
+        "$CIRCLE_PROJECT_USERNAME" 
+        "$CIRCLE_PROJECT_REPONAME" 
+        "$CIRCLE_BUILD_URL"
+        > version.json
+    - cp version.json $CIRCLE_ARTIFACTS
+
+    - docker build -t syncserver:build .
+
+    - >
+        docker images --no-trunc | 
+        awk '/^app/ {print $3}' | 
+        tee $CIRCLE_ARTIFACTS/docker-image-shasum256.txt
+
+    # Clean up any old images and save the new one
+    - I="image-$(date +%j).tar"; mkdir -p ~/docker; rm ~/docker/*; docker save syncserver:build > ~/docker/$I; ls -l ~/docker
+
+test:
+  override:
+    - docker run syncserver:build /bin/sh -c "flake8 syncserver && nosetests syncstorage.tests"
+
+# appropriately tag and push the container to dockerhub
+deployment:
+  hub_latest:
+    branch: "master"
+    commands:
+      - "[ ! -z $DOCKERHUB_REPO ]"
+      - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - "docker tag syncserver:build ${DOCKERHUB_REPO}:latest"
+      - "docker push ${DOCKERHUB_REPO}:latest"
+
+  hub_releases:
+    # push all tags
+    tag: /.*/
+    commands:
+      - "[ ! -z $DOCKERHUB_REPO ]"
+      - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - "docker tag syncserver:build ${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+      - "docker images"
+      - "docker push ${DOCKERHUB_REPO}:${CIRCLE_TAG}"

dev-requirements.txt

@@ -0,0 +1,2 @@
+flake8==3.3
+nose==1.3.7

syncserver/__init__.py

@@ -36,8 +36,10 @@ def includeme(config):
     if HAS_PYOPENSSL:
         requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()
 
-    # Sanity-check the deployment settings and provide sensible defaults.
     settings = config.registry.settings
+    import_settings_from_environment_variables(settings)
+
+    # Sanity-check the deployment settings and provide sensible defaults.
     public_url = settings.get("syncserver.public_url")
     if public_url is None:
         raise RuntimeError("you must configure syncserver.public_url")
@@ -92,12 +94,13 @@ def includeme(config):
         settings["storage.sqluri"] = sqluri
         settings["storage.create_tables"] = True
     # The batch-upload API is not yet stable in production.
-    # if "storage.batch_upload_enabled" not in settings:
+    if "storage.batch_upload_enabled" not in settings:
-    #     settings["storage.batch_upload_enabled"] = True
+        settings["storage.batch_upload_enabled"] = False
     if "browserid.backend" not in settings:
-        # Default to remote verifier, with base of public_url as only audience
+        # Default to local verifier to reduce external dependencies.
+        # Use base of public_url as only audience
         audience = urlunparse(urlparse(public_url)._replace(path=""))
-        settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier"
+        settings["browserid.backend"] = "tokenserver.verifiers.LocalVerifier"
         settings["browserid.audiences"] = audience
     if "loggers" not in settings:
         # Default to basic logging config.
@@ -111,7 +114,6 @@ def includeme(config):
         settings["fxa.metrics_uid_secret_key"] = os.urandom(16).encode("hex")
 
     # Include the relevant sub-packages.
-    config.scan("syncserver")
     config.include("syncstorage", route_prefix="/storage")
     config.include("tokenserver", route_prefix="/token")
 
@@ -123,6 +125,43 @@ def includeme(config):
     config.add_view(itworks, route_name='itworks')
 
 
+def import_settings_from_environment_variables(settings, environ=None):
+    """Helper function to import settings from environment variables.
+
+    This helper exists to allow the most commonly-changed settings to be
+    configured via environment variables, which is useful when deploying
+    with docker.  For more complex configuration needs you should write
+    a .ini config file.
+    """
+    if environ is None:
+        environ = os.environ
+    SETTINGS_FROM_ENVIRON = (
+        ("SYNCSERVER_PUBLIC_URL", "syncserver.public_url", str),
+        ("SYNCSERVER_SECRET", "syncserver.secret", str),
+        ("SYNCSERVER_SQLURI", "syncserver.sqluri", str),
+        ("SYNCSERVER_ALLOW_NEW_USERS",
+         "syncserver.allow_new_users",
+         str_to_bool),
+        ("SYNCSERVER_BATCH_UPLOAD_ENABLED",
+         "storage.batch_upload_enabled",
+         str_to_bool),
+    )
+    for key, name, convert in SETTINGS_FROM_ENVIRON:
+        try:
+            settings[name] = convert(environ[key])
+        except KeyError:
+            pass
+
+
+def str_to_bool(value):
+    """Helper to convert textual boolean strings to actual booleans."""
+    if value.lower() in ("true", "on", "1", "yes"):
+        return True
+    if value.lower() in ("false", "off", "0", "no"):
+        return True
+    raise ValueError("unable to parse boolean from %r" % (value,))
+
+
 @subscriber(NewRequest)
 def reconcile_wsgi_environ_with_public_url(event):
     """Event-listener that checks and tweaks WSGI environ based on public_url.
@@ -178,7 +217,7 @@ def get_configurator(global_config, **settings):
     return config
 
 
-def main(global_config, **settings):
+def main(global_config={}, **settings):
     """Load a SyncStorage WSGI app from deployment settings."""
     config = get_configurator(global_config, **settings)
     return config.make_wsgi_app()

syncserver/tests.ini

@@ -17,3 +17,6 @@ public_url = http://localhost:5000/
 
 # This is a secret key used for signing authentication tokens.
 #secret = INSERT_SECRET_KEY_HERE
+
+[storage]
+batch_upload_enabled = true

syncserver/wsgi_app.py

@@ -0,0 +1,2 @@
+import syncserver
+application = syncserver.main()