Helpers to compare settings and requirements when migrating from a regular project to a divio cloud project

key_requirements_compare.py

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import pip
import pip.req
import uuid


# from pprint import pprint as pp; import key_requirements_compare; reload(key_requirements_compare); compared=key_requirements_compare.tst(); print(len(compared['old']))


ignore_packages = (
    'gunicorn',
    'newrelic',
    # dploi fabric related
    'fabric',
    'dploi-fabric',
    'divio-dploi-deployments',
    'paramiko',
    'pycrypto',
    'ecdsa',
    # other
    # 'django-appmedia',  # this is outdated and unnesecary. I hope.
    # 'django-commonsearch', # solr specific
    # 'amqplib',  # does not seem to be needed by celery for amqp anymore.
    # 'django-gsa',  # We don't use the Google Search Appliance anymore.
    'django-localdevstorage',  # We're just using the S3 storage directly
    'wheel',

)


def load_requirements(req_file):
    return {
        itm.name.lower(): itm
        for itm in pip.req.parse_requirements(
            req_file,
            session=uuid.uuid4().hex,
        )
        if itm.name
    }


def compare(old_requirements, new_requirements):
    old_reqs = load_requirements(old_requirements)
    new_reqs = load_requirements(new_requirements)
    # return compare_packages(
    #     old=old_reqs.keys(),
    #     new=new_reqs.keys(),
    # )
    return compare_packages_and_versions(
        old_reqs=old_reqs,
        new_reqs=new_reqs,
    )

def compare_packages(old, new):
    old = set(old) - set(ignore_packages)
    new = set(new) - set(ignore_packages)
    return {
        'common': old.intersection(new),
        'old': old - new,
        'new': new - old,
    }

def compare_packages_and_versions(old_reqs, new_reqs):
    old = set(old_reqs.keys()) - set(ignore_packages)
    new = set(new_reqs.keys()) - set(ignore_packages)
    return {
        'common': add_versions(
            names=old.intersection(new),
            old_reqs=old_reqs,
            new_reqs=new_reqs,
        ),
        'old': add_versions(
            names=(old - new),
            old_reqs=old_reqs,
            new_reqs=new_reqs,
        ),
        'new': add_versions(
            names=(new - old),
            old_reqs=old_reqs,
            new_reqs=new_reqs,
        ),
    }

def add_versions(names, old_reqs, new_reqs):
    result = {}
    for name in names:
        result[name] = {
            'old': old_reqs.get(name, None),
            'new': new_reqs.get(name, None),
        }
    return result


def version_table(reqs):
    # TODO: highlight minor and major version bumps
    # TODO: checklist mode to save for every dependency if the version bump is
    #       is ok
    from terminaltables import AsciiTable
    from pkg_resources import SetuptoolsVersion
    names = sorted(reqs.keys())
    table_data = [
        ['Package', 'Old Version', '', 'New Version'],
    ]
    for name in names:
        name = name
        info = reqs.get(name)

        old_version = specifier_to_version_string(info['old'].specifier) if info['old'] else ''
        new_version = specifier_to_version_string(info['new'].specifier) if info['new'] else ''
        if old_version and new_version:
            old_v = SetuptoolsVersion(old_version)
            new_v = SetuptoolsVersion(new_version)
            if old_v == new_v:
                diff = '=='
            elif old_v > new_v:
                diff = ' >  [!]'
            elif old_v < new_v:
                diff = ' <'
            else:
                diff = '?'
        else:
            diff = ''
        table_data.append([name, old_version, diff, new_version])
    return AsciiTable(table_data)


def specifier_to_version_string(spec):
    if len(spec._specs) == 0:
        return ''
    assert len(spec._specs) == 1
    spec1 = list(spec._specs)[0]
    assert spec1.operator == '=='
    return spec1.version


def tst():
    compared = compare(
        old_requirements='key_requirements.txt',
        new_requirements='requirements.txt',
    )
    for key, reqs in compared.items():
        print('')
        print('===== {} ====='.format(key))
        table = version_table(reqs)
        print(table.table)
    return compared

key_settings_compare.py

# -*- coding: utf-8 -*-
from __future__ import unicode_literals


# DEBUG: compare key settings
from pprint import pprint as pp
from django.conf import settings
import json

old_data = json.load(open('key_settings.json', 'r'))

def compare(name, old, new):
    print
    print '='*10
    print name
    print '='*10
    old = set(old)
    new = set(new)

    print "common:".upper()
    pp(sorted(list(new.intersection(old))))

    print "only in old:".upper()
    pp(sorted(list(old.difference(new))))

    print "only in new:".upper()
    pp(sorted(list(new.difference(old))))


def run():
    compare('INSTALLED_APPS', old_data['INSTALLED_APPS'], settings.INSTALLED_APPS)
    compare('MIDDLEWARE_CLASSES', old_data['MIDDLEWARE_CLASSES'], settings.MIDDLEWARE_CLASSES)
    compare('context_processors', old_data['TEMPLATES'][0]['OPTIONS']['context_processors'], settings.TEMPLATES[0]['OPTIONS']['context_processors'])

key_settings_extract.py

from django.conf import settings
import json


data = {
    'INSTALLED_APPS': settings.INSTALLED_APPS,
    'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES,
    'TEMPLATES': settings.TEMPLATES,
}

print json.dumps(data, indent=4)