From d0d47fae3b5b542e7120966d83a927c97dca897a Mon Sep 17 00:00:00 2001 From: Ryan Kelly Date: Thu, 6 Feb 2014 17:29:34 +1100 Subject: [PATCH] Initial commit; refactored from server-full2 codebase --- .gitignore | 8 ++++ MANIFEST.in | 3 ++ Makefile | 37 +++++++++++++++++ README.rst | 73 +++++++++++++++++++++++++++++++++ requirements.txt | 60 +++++++++++++++++++++++++++ setup.py | 16 ++++++++ syncserver.ini | 17 ++++++++ syncserver.wsgi | 40 ++++++++++++++++++ syncserver/__init__.py | 92 ++++++++++++++++++++++++++++++++++++++++++ syncserver/tests.ini | 17 ++++++++ 10 files changed, 363 insertions(+) create mode 100644 .gitignore create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.rst create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 syncserver.ini create mode 100644 syncserver.wsgi create mode 100644 syncserver/__init__.py create mode 100644 syncserver/tests.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..813f880 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.pyc +*.mako.py +local +*.egg-info +*.swp +\.coverage +*~ +nosetests.xml diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d5be2d1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +syncserver.ini +syncserver.wsgi +syncserver/tests.ini diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d6e61cd --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +VIRTUALENV = virtualenv +PYTHON = local/bin/python +NOSE = local/bin/nosetests -s +FLAKE8 = local/bin/flake8 +PIP = local/bin/pip +PIP_CACHE = /tmp/pip-cache.${USER} +BUILD_TMP = /tmp/syncstorage-build.${USER} +PYPI = https://pypi.python.org/simple +INSTALL = $(PIP) install -U -i $(PYPI) + +.PHONY: all build test serve clean + +all: build test + +build: + $(VIRTUALENV) --no-site-packages --distribute ./local + $(INSTALL) --upgrade Distribute + $(INSTALL) pip + $(INSTALL) nose + $(INSTALL) flake8 + $(INSTALL) -r requirements.txt + $(PYTHON) ./setup.py develop + +test: + # Basic syntax and sanity checks. + $(FLAKE8) ./syncserver + # Testcases from the bundled apps. + $(NOSE) syncstorage.tests + $(NOSE) tokenserver.tests + # Live tests against a running server. + ./local/bin/pserve syncserver/tests.ini & SERVER_PID=$$! ; sleep 2 ; ./local/bin/python -m syncstorage.tests.functional.test_storage --use-token-server http://localhost:5000/token/1.0/sync/1.5 ; kill $$SERVER_PID + +serve: + ./local/bin/pserve ./syncserver.ini + +clean: + rm -rf ./local diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..6ae6791 --- /dev/null +++ b/README.rst @@ -0,0 +1,73 @@ +Run-Your-Own Firefox Sync Server +================================ + +This is an all-in-one package for running a self-hosted Firefox Sync server. +If bundles the "tokenserver" project for authentication and the "syncstorage" +project for storage, produce a single stand-alone webapp. + +Complete installation instructions are available at: + + https://docs.services.mozilla.com/howtos/run-sync-1.5.html + + +Quickstart +---------- + +The Sync Server software runs using **python 2.6** or later, and the build +process requires **make** and **virtualenv**. You will need to have the +following packages (or similar, depending on your operating system) installed: + +- python2.7 +- python2.7-dev +- python-virtualenv +- make + +Take a checkout of this repository, then run "make build" to pull in the +necessary python package dependencies:: + + $ git clone https://github.com/mozilla-services/syncserver + $ cd syncserver + $ make build + +To sanity-check that things got installed correctly, do the following:: + + $ make test + +Now you can run the server:: + + $ make serve + +This should start a server on http://localhost:5000/. + +Now go into Firefox's `about:config` page, search for a setting named +"tokenServerURI", and change it to point to your server:: + + services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5 + +Firefox should now sync against your local server rather than the default +Mozilla-hosted servers. + +For more details on setting up a stable deployment, see: + + https://docs.services.mozilla.com/howtos/run-sync-1.5.html + + +Customization +------------- + +All customization of the server can be done by editing the file +"syncserver.ini", which contains lots of comments to help you on +your way. Things you might like to change include: + + * The client-visible hostname for your server. Edit the "public_url" + key under the [syncstorage] section. + + * The database in which to store sync data. Edit the "sqluri" setting + under the [syncstorage] section. + + +Questions, Feedback +------------------- + +- IRC channel: #sync. See http://irc.mozilla.org/ +- Mailing list: https://mail.mozilla.org/admin/services-dev diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7db9ab8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,60 @@ +beautifulsoup4==4.3.2 +cef==0.5 +Chameleon==2.14 +chardet==2.2.1 +circus==0.10 +cornice==0.16.2 +coverage==3.7.1 +docutils==0.11 +flake8==2.1.0 +gevent==1.0 +greenlet==0.4.1 +hawkauthlib==0.1.1 +Jinja2==2.7.1 +Mako==0.9.1 +MarkupSafe==0.18 +mccabe==0.2.1 +metlog-py==0.10 +ordereddict==1.1 +Paste==1.7.5.1 +PasteDeploy==1.5.2 +PasteScript==1.7.5 +pep8==1.4.6 +powerhose==0.7 +protobuf==2.5 +psutil==1.2.1 +PyBrowserID==0.9.1 +pyflakes==0.7.3 +Pygments==1.6 +PyMySQL==0.6.1 +pymysql_sa==1.0 +pyramid==1.4.3 +pyramid_debugtoolbar==1.0.9 +pyramid-hawkauth==0.1.0 +pyramid_jinja2==1.9 +repoze.lru==0.6 +requests==2.1 +simplejson==3.3.2 +six==1.5.2 +Sphinx==1.2 +SQLAlchemy==0.9.1 +tokenlib==0.3.1 +translationstring==1.1 +umemcache==1.6.3 +unittest2==0.5.1 +uWSGI==1.9.20 +venusian==1.0a8 +waitress==0.8.8 +WebOb==1.3.1 +WebTest==2.0.11 +wsgi_intercept==0.6 +wsgiref==0.1.2 +zope.component==4.1.0 +zope.deprecation==4.1.0 +zope.event==4.0.2 +zope.interface==4.0.5 +https://argparse.googlecode.com/files/argparse-1.2.1.tar.gz +http://github.com/mozilla-services/mozservices/archive/master.zip +https://github.com/mozilla-services/wimms/archive/master.zip +https://github.com/mozilla-services/tokenserver/archive/master.zip +http://github.com/mozilla-services/server-syncstorage/archive/master.zip diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4e53abb --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +from setuptools import setup + +entry_points = """ +[paste.app_factory] +main = syncserver:main +""" + +setup( + name='syncserver', + version="1.5", + packages=['syncserver'], + entry_points=entry_points +) diff --git a/syncserver.ini b/syncserver.ini new file mode 100644 index 0000000..011914e --- /dev/null +++ b/syncserver.ini @@ -0,0 +1,17 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:syncserver + +[syncserver] +# This must be edited to point to the public URL of your server. +public_url = http://localhost:5000/ + +# This defines the database in which to store all server data. +#sqluri = sqlite:////tmp/syncserver.db + +# This is a secret key used for signing authentication tokens. +#secret = INSERT_SECRET_KEY_HERE diff --git a/syncserver.wsgi b/syncserver.wsgi new file mode 100644 index 0000000..2499dfe --- /dev/null +++ b/syncserver.wsgi @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +import site +from logging.config import fileConfig +from ConfigParser import NoSectionError + +# detecting if virtualenv was used in this dir +_CURDIR = os.path.dirname(os.path.abspath(__file__)) +_PY_VER = sys.version.split()[0][:3] +_SITE_PKG = os.path.join(_CURDIR, 'lib', 'python' + _PY_VER, 'site-packages') + +# adding virtualenv's site-package and ordering paths +saved = sys.path[:] + +if os.path.exists(_SITE_PKG): + site.addsitedir(_SITE_PKG) + +for path in sys.path: + if path not in saved: + saved.insert(0, path) + +sys.path[:] = saved + +# setting up the egg cache to a place where apache can write +os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs' + +# setting up logging +ini_file = os.path.join(_CURDIR, 'syncserver.ini') +try: + fileConfig(ini_file) +except NoSectionError: + pass + +# running the app using Paste +from paste.deploy import loadapp +application = loadapp('config:%s'% ini_file) diff --git a/syncserver/__init__.py b/syncserver/__init__.py new file mode 100644 index 0000000..0e8e05f --- /dev/null +++ b/syncserver/__init__.py @@ -0,0 +1,92 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from pyramid.response import Response +import mozsvc.config + + +def includeme(config): + """Install SyncServer application into the given Pyramid configurator.""" + # Set the umask so that files are created with secure permissions. + # Necessary for e.g. created-on-demand sqlite database files. + os.umask(0077) + + # Sanity-check the deployment settings and provide sensible defaults. + settings = config.registry.settings + public_url = settings.get("syncserver.public_url") + if public_url is None: + raise RuntimeError("you much configure syncserver.public_url") + secret = settings.get("syncserver.secret") + if secret is None: + secret = os.urandom(32).encode("hex") + sqluri = settings.get("syncserver.sqluri") + if sqluri is None: + sqluri = "sqlite:///:memory:" + + # Configure app-specific defaults based on top-level configuration. + settings.pop("config", None) + if "tokenserver.backend" not in settings: + # Default to sql node-assignment backend + settings["tokenserver.backend"] =\ + "tokenserver.assignment.sqlnode.SQLNodeAssignment" + settings["tokenserver.sqluri"] = sqluri + if "tokenserver.applications" not in settings: + # Default to just the sync-1.5 application + settings["tokenserver.applications"] = "sync-1.5" + if "tokenserver.secrets.backend" not in settings: + # Default to a single fixed signing secret + settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets" + settings["tokenserver.secrets.secrets"] = [secret] + if "hawkauth.secrets.backend" not in settings: + # Default to the same secrets backend as the tokenserver + for key in settings.keys(): + if key.startswith("tokenserver.secrets."): + newkey = "hawkauth" + key[len("tokenserver"):] + settings[newkey] = settings[key] + if "storage.backend" not in settings: + # Default to sql syncstorage backend + settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage" + settings["storage.sqluri"] = sqluri + settings["storage.create_tables"] = True + if "browserid.backend" not in settings: + # Default to remote verifier, with public_url as only audience + settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier" + settings["browserid.audiences"] = public_url + if "metlog.backend" not in settings: + # Default to logging to stdout + settings["metlog.backend"] = "mozsvc.metrics.MetlogPlugin" + settings["metlog.enabled"] = True + settings["metlog.sender_class"] = "metlog.senders.StdOutSender" + if "cef.use" not in settings: + # Default to no CEF logging + settings["cef.use"] = False + + # Include the relevant sub-packages. + config.include("syncstorage", route_prefix="/storage") + config.include("tokenserver", route_prefix="/token") + + # Add a top-level "it works!" view. + def itworks(request): + return Response("it works!") + + config.add_route('itworks', '/') + config.add_view(itworks, route_name='itworks') + + +def get_configurator(global_config, **settings): + """Load a SyncStorge configurator object from deployment settings.""" + config = mozsvc.config.get_configurator(global_config, **settings) + config.begin() + try: + config.include(includeme) + finally: + config.end() + return config + + +def main(global_config, **settings): + """Load a SyncStorage WSGI app from deployment settings.""" + config = get_configurator(global_config, **settings) + return config.make_wsgi_app() diff --git a/syncserver/tests.ini b/syncserver/tests.ini new file mode 100644 index 0000000..50b7776 --- /dev/null +++ b/syncserver/tests.ini @@ -0,0 +1,17 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:SyncServer + +[syncserver] +# This must be edited to point to the public URL of your server. +public_url = http://localhost:5000/ + +# This defines the database in which to store all server data. +#sqluri = sqlite:////tmp/syncserver.db + +# This is a secret key used for signing authentication tokens. +#secret = INSERT_SECRET_KEY_HERE