freider

class PigJob(luigi.Task):
    def script_body(self):
        """
        -- Your pig script body goes here
        """

    def requires(self):
        # should return a dictionary of input tasks
        # they will be inserted as variables in the beginning of the produced
        # pig script

freider

class SmartPostgresTarget(PostgresTarget):
    """PostgresTarget with possibility to specify an existence query"""
    def __init__(self, host, database, user, password,
                 table,
                 exists_query,
                 update_id=None):
        update_id = update_id or exists_query
        self.exists_query = exists_query
        super(SmartPostgresTarget, self).__init__(
            host, database, user, password,

freider

import functools
from luigi import LocalTarget
from luigi.format import Format


class PipeWrapper(object):
    """Wrap pipe in a "real" Python object in case it's a `file`
    """
    # TODO: build this into luigi and use it by default for LocalFile.open('r')
    def __init__(self, pipe):