tests and robot tests framework, build overhaul

env

env

engines.cfg

engines.cfg

.installed.cfg

setup.cfg

*.pyc

*.pyc

*/*.pyc

*/*.pyc

bin/

include/

lib/

build/

develop-eggs/

parts/

# convenience makefile to boostrap & run buildout

# use `make options=-v` to run buildout with extra options

version = 2.7

python = bin/python

options =

all: .installed.cfg

.installed.cfg: bin/buildout buildout.cfg setup.py

	bin/buildout $(options)

bin/buildout: $(python) buildout.cfg bootstrap.py

	$(python) bootstrap.py

	@touch $@

$(python):

	virtualenv -p python$(version) --no-site-packages .

	@touch $@

tests: .installed.cfg

	@bin/test

enginescfg:

	@test -f ./engines.cfg || echo "Copying engines.cfg ..."

	@cp --no-clobber engines.cfg_sample engines.cfg

robot: .installed.cfg enginescfg

	@bin/robot

flake8: .installed.cfg

	@bin/flake8 setup.py

	@bin/flake8 ./searx/

coverage: .installed.cfg

	@bin/coverage run --source=./searx/ --branch bin/test

	@bin/coverage report --show-missing

	@bin/coverage html --directory ./coverage

minimal: bin/buildout production.cfg setup.py enginescfg

	bin/buildout -c production.cfg $(options)

	@echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`"

	@echo "* Hint 1: on production, disable debug mode and change secret_key"

	@echo "* Hint 2: to run server execute 'bin/searx-run'"

clean:

	@rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs \

		searx.egg-info lib include .coverage coverage

.PHONY: all tests enginescfg robot flake8 coverage minimal clean

For all the details, follow this [step by step installation](https://github.com/asciimoo/searx/wiki/Installation)

For all the details, follow this [step by step installation](https://github.com/asciimoo/searx/wiki/Installation)

### Alternative (Recommended) Installation

* clone source: `git clone git@github.com:asciimoo/searx.git && cd searx`

* build in current folder: `make minimal`

* run `bin/searx-run` to start the application

### Development

Just run `make`. Versions of dependencies are pinned down inside `versions.cfg` to produce most stable build.

#### Command make

##### `make`

Builds development environment with testing support.

##### `make tests`

Runs tests. You can write tests [here](https://github.com/asciimoo/searx/tree/master/searx/tests) and remember 'untested code is broken code'.

##### `make robot`

Runs robot (Selenium) tests, you must have `firefox` installed because this functional tests actually run the browser and perform operations on it. Also searx is executed with [settings_robot](https://github.com/asciimoo/searx/blob/master/searx/settings_robot.py).

##### `make flake8`

'pep8 is a tool to check your Python code against some of the style conventions in [PEP 8](http://www.python.org/dev/peps/pep-0008/).'

##### `make coverage`

Checks coverage of tests, after running this, execute this: `firefox ./coverage/index.html`

##### `make minimal`

Used to make co-called production environment - without tests (you should ran tests before deploying searx on the server).

##### `make clean`

Deletes several folders and files (see `Makefile` for more), so that next time you run any other `make` command it will rebuild everithing.

### TODO

### TODO

* Moar engines

* Moar engines

* Language support

* Language support

* Documentation

* Documentation

* Pagination

* Pagination

* Fix `flake8` errors, `make flake8` will be merged into `make tests` when it does not fail anymore

* Tests

* Tests

* When we have more tests, we can integrate Travis-CI

### Bugs

### Bugs

[buildout]

extends = versions.cfg

versions = versions

unzip = true

newest = false

extends = versions.cfg

versions = versions

prefer-final = true

develop = .

extensions =

    buildout_versions

eggs =

    searx

parts =

    omelette

[omelette]

recipe = collective.recipe.omelette

eggs = ${buildout:eggs}

##############################################################################

#

# Copyright (c) 2006 Zope Foundation and Contributors.

# All Rights Reserved.

#

# This software is subject to the provisions of the Zope Public License,

# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.

# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED

# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS

# FOR A PARTICULAR PURPOSE.

#

##############################################################################

"""Bootstrap a buildout-based project

Simply run this script in a directory containing a buildout.cfg.

The script accepts buildout command-line options, so you can

use the -c option to specify an alternate configuration file.

"""

import os, shutil, sys, tempfile, urllib, urllib2, subprocess

from optparse import OptionParser

if sys.platform == 'win32':

    def quote(c):

        if ' ' in c:

            return '"%s"' % c  # work around spawn lamosity on windows

        else:

            return c

else:

    quote = str

# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments.

stdout, stderr = subprocess.Popen(

    [sys.executable, '-Sc',

     'try:\n'

     '    import ConfigParser\n'

     'except ImportError:\n'

     '    print 1\n'

     'else:\n'

     '    print 0\n'],

    stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()

has_broken_dash_S = bool(int(stdout.strip()))

# In order to be more robust in the face of system Pythons, we want to

# run without site-packages loaded.  This is somewhat tricky, in

# particular because Python 2.6's distutils imports site, so starting

# with the -S flag is not sufficient.  However, we'll start with that:

if not has_broken_dash_S and 'site' in sys.modules:

    # We will restart with python -S.

    args = sys.argv[:]

    args[0:0] = [sys.executable, '-S']

    args = map(quote, args)

    os.execv(sys.executable, args)

# Now we are running with -S.  We'll get the clean sys.path, import site

# because distutils will do it later, and then reset the path and clean

# out any namespace packages from site-packages that might have been

# loaded by .pth files.

clean_path = sys.path[:]

import site  # imported because of its side effects

sys.path[:] = clean_path

for k, v in sys.modules.items():

    if k in ('setuptools', 'pkg_resources') or (

        hasattr(v, '__path__') and

        len(v.__path__) == 1 and

        not os.path.exists(os.path.join(v.__path__[0], '__init__.py'))):

        # This is a namespace package.  Remove it.

        sys.modules.pop(k)

is_jython = sys.platform.startswith('java')

setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'

distribute_source = 'http://python-distribute.org/distribute_setup.py'

distribute_source = 'https://bitbucket.org/pypa/setuptools/raw/f657df1f1ed46596d236376649c99a470662b4ba/distribute_setup.py'

# parsing arguments

def normalize_to_url(option, opt_str, value, parser):

    if value:

        if '://' not in value:  # It doesn't smell like a URL.

            value = 'file://%s' % (

                urllib.pathname2url(

                    os.path.abspath(os.path.expanduser(value))),)

        if opt_str == '--download-base' and not value.endswith('/'):

            # Download base needs a trailing slash to make the world happy.

            value += '/'

    else:

        value = None

    name = opt_str[2:].replace('-', '_')

    setattr(parser.values, name, value)

usage = '''\

[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]

Bootstraps a buildout-based project.

Simply run this script in a directory containing a buildout.cfg, using the

Python that you want bin/buildout to use.

Note that by using --setup-source and --download-base to point to

local resources, you can keep this script from going over the network.

'''

parser = OptionParser(usage=usage)

parser.add_option("-v", "--version", dest="version",

                          help="use a specific zc.buildout version")

parser.add_option("-d", "--distribute",

                   action="store_true", dest="use_distribute", default=False,

                   help="Use Distribute rather than Setuptools.")

parser.add_option("--setup-source", action="callback", dest="setup_source",

                  callback=normalize_to_url, nargs=1, type="string",

                  help=("Specify a URL or file location for the setup file. "

                        "If you use Setuptools, this will default to " +

                        setuptools_source + "; if you use Distribute, this "

                        "will default to " + distribute_source + "."))

parser.add_option("--download-base", action="callback", dest="download_base",

                  callback=normalize_to_url, nargs=1, type="string",

                  help=("Specify a URL or directory for downloading "

                        "zc.buildout and either Setuptools or Distribute. "

                        "Defaults to PyPI."))

parser.add_option("--eggs",

                  help=("Specify a directory for storing eggs.  Defaults to "

                        "a temporary directory that is deleted when the "

                        "bootstrap script completes."))

parser.add_option("-t", "--accept-buildout-test-releases",

                  dest='accept_buildout_test_releases',

                  action="store_true", default=False,

                  help=("Normally, if you do not specify a --version, the "

                        "bootstrap script and buildout gets the newest "

                        "*final* versions of zc.buildout and its recipes and "

                        "extensions for you.  If you use this flag, "

                        "bootstrap and buildout will get the newest releases "

                        "even if they are alphas or betas."))

parser.add_option("-c", None, action="store", dest="config_file",

                   help=("Specify the path to the buildout configuration "

                         "file to be used."))

options, args = parser.parse_args()

if options.eggs:

    eggs_dir = os.path.abspath(os.path.expanduser(options.eggs))

else:

    eggs_dir = tempfile.mkdtemp()

if options.setup_source is None:

    if options.use_distribute:

        options.setup_source = distribute_source

    else:

        options.setup_source = setuptools_source

if options.accept_buildout_test_releases:

    args.insert(0, 'buildout:accept-buildout-test-releases=true')

try:

    import pkg_resources

    import setuptools  # A flag.  Sometimes pkg_resources is installed alone.

    if not hasattr(pkg_resources, '_distribute'):

        raise ImportError

except ImportError:

    ez_code = urllib2.urlopen(

        options.setup_source).read().replace('\r\n', '\n')

    ez = {}

    exec ez_code in ez

    setup_args = dict(to_dir=eggs_dir, download_delay=0)

    if options.download_base:

        setup_args['download_base'] = options.download_base

    if options.use_distribute:

        setup_args['no_fake'] = True

        if sys.version_info[:2] == (2, 4):

            setup_args['version'] = '0.6.32'

    ez['use_setuptools'](**setup_args)

    if 'pkg_resources' in sys.modules:

        reload(sys.modules['pkg_resources'])

    import pkg_resources

    # This does not (always?) update the default working set.  We will

    # do it.

    for path in sys.path:

        if path not in pkg_resources.working_set.entries:

            pkg_resources.working_set.add_entry(path)

cmd = [quote(sys.executable),

       '-c',

       quote('from setuptools.command.easy_install import main; main()'),

       '-mqNxd',

       quote(eggs_dir)]

if not has_broken_dash_S:

    cmd.insert(1, '-S')

find_links = options.download_base

if not find_links:

    find_links = os.environ.get('bootstrap-testing-find-links')

if not find_links and options.accept_buildout_test_releases:

    find_links = 'http://downloads.buildout.org/'

if find_links:

    cmd.extend(['-f', quote(find_links)])

if options.use_distribute:

    setup_requirement = 'distribute'

else:

    setup_requirement = 'setuptools'

ws = pkg_resources.working_set

setup_requirement_path = ws.find(

    pkg_resources.Requirement.parse(setup_requirement)).location

env = dict(

    os.environ,

    PYTHONPATH=setup_requirement_path)

requirement = 'zc.buildout'

version = options.version

if version is None and not options.accept_buildout_test_releases:

    # Figure out the most recent final version of zc.buildout.

    import setuptools.package_index

    _final_parts = '*final-', '*final'

    def _final_version(parsed_version):

        for part in parsed_version:

            if (part[:1] == '*') and (part not in _final_parts):

                return False

        return True

    index = setuptools.package_index.PackageIndex(

        search_path=[setup_requirement_path])

    if find_links:

        index.add_find_links((find_links,))

    req = pkg_resources.Requirement.parse(requirement)

    if index.obtain(req) is not None:

        best = []

        bestv = None

        for dist in index[req.project_name]:

            distv = dist.parsed_version

            if distv >= pkg_resources.parse_version('2dev'):

                continue

            if _final_version(distv):

                if bestv is None or distv > bestv:

                    best = [dist]

                    bestv = distv

                elif distv == bestv:

                    best.append(dist)

        if best:

            best.sort()

            version = best[-1].version

if version:

    requirement += '=='+version

else:

    requirement += '<2dev'

cmd.append(requirement)

if is_jython:

    import subprocess

    exitcode = subprocess.Popen(cmd, env=env).wait()

else:  # Windows prefers this, apparently; otherwise we would prefer subprocess

    exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))

if exitcode != 0:

    sys.stdout.flush()

    sys.stderr.flush()

    print ("An error occurred when trying to install zc.buildout. "

           "Look above this message for any errors that "

           "were output by easy_install.")

    sys.exit(exitcode)

ws.add_entry(eggs_dir)

ws.require(requirement)

import zc.buildout.buildout

# If there isn't already a command in the args, add bootstrap

if not [a for a in args if '=' not in a]:

    args.append('bootstrap')

# if -c was provided, we push it back into args for buildout's main function

if options.config_file is not None:

    args[0:0] = ['-c', options.config_file]

zc.buildout.buildout.main(args)

if not options.eggs:  # clean up temporary egg directory

    shutil.rmtree(eggs_dir)