Git Log Parser & Aggregator

git-log-parser.js

/* eslint-env node */
/* eslint-disable no-console, dot-notation */

const fse = require('fs-extra');
const readline = require('readline');

// const platform = 'mw_less';
// const platform = 'mw_scss';
const platform = 'cosmos';

console.log(`Processing git-log file for ${platform} started...`);

const commits = [];
let commit = false;

const lineReader = readline.createInterface({
    input: fse.createReadStream(`data-raw/git-log-stat-${platform}.txt`)
});

lineReader.on('line', function (line) {
    // ignore empty lines
    if (line === '' || line === '\n') {
        return;
    }
    // we've found a new commit!
    if (line.match(/^commit/)) {
        // push the previous commit in the store (if is not the initial "false" commit)
        if(commit) {
            commits.push(commit);
        }
        // initialise a new commit
        commit = {
            merge: false,
            exclude: false,
        };
        // get the hash value
        commit.hash = line.match(/^commit (.*)/)[1];
        // console.log('Found hash:', commit.hash);
    }
    if (line.match(/^Author:/)) {
        // get the author name
        commit.author = line.match(/^Author: (.*) <(.*)>/)[1];
        // console.log('Found author:', commit.author);
    }
    if (line.match(/^Date:/)) {
        // get the commit date
        commit.date = line.match(/^Date:\s+(.*)/)[1];
        // console.log('Found date:', commit.date);
    }
    if (line.match(/^Merge:/)) {
        // is a merge
        commit.merge = true;
        // console.log('Found a merge');
    }
    if (line.match(/^    [^\#]/)) {
        // get the (first line of) commit message
        if(!commit.message) {
            commit.message = line.match(/^    (.*)/)[1];
        }
        // console.log('Found a message', commit.message);
    }
    if (line.match(/\d+ file[s]? changed/)) { // in case of merge this line doesn't exist
        // get the file changes => X files changed, Y insertions(+), Z deletions(-)
        line.split(', ').forEach(function(change) {
            if(change.match(/\d+ file[s]? changed/)) {
                commit.files = change.match(/(\d+) file[s]? changed/)[1];
                // console.log('Files changed', commit.files);
            }
            if(change.match(/\d+ insertion[s]?\(\+\)/)) {
                commit.insertions = change.match(/(\d+) insertion[s]?\(\+\)/)[1];
                // console.log('Insertions', commit.insertions);
            }
            if(change.match(/\d+ deletion[s]?\(-\)/)) {
                commit.deletions = change.match(/(\d+) deletion[s]?\(-\)/)[1];
                // console.log('deletions', commit.deletions);
            }
        });
    }
});

lineReader.on('close', () => {

    // write commits data as tab-separated values
    let output = '';
    commits.forEach(function(commit) {
        const hash = commit.hash;
        const author = commit.author;
        const date = commit.date;
        const merge = commit.merge ? 1 : 0;
        const message = commit.message;
        const files = commit.files ? parseInt(commit.files, 10) : 0;
        const insertions = commit.insertions ? parseInt(commit.insertions, 10) : 0;
        const deletions = commit.deletions ? parseInt(commit.deletions, 10) : 0;
        const row = `${hash}\t${author}\t${date}\t${merge}\t${message}\t${files}\t${insertions}\t${deletions}`;
        output += `${row}\n`;
        // console.log(row);
    });
    fse.writeFileSync(`./data-processed/commits-${platform}.tsv`, output);

    // write commits data as JSON file
    fse.writeJsonSync(`./data-processed/commits-${platform}.json`, commits);

    // write aggregated commits data as JSON file
    let aggregated = {};
    commits.forEach(function(commit) {
        if (commit.merge) {
            return;
        }

        // blacklisting
        if(platform === 'mw_less') {
            if(
                // initial project setup
                commit.message.match(/^\[rework\]/) ||
                ...
                // remove less codebase
                commit.message.match(/^\[MW-****\]/)
            ) {
                return;
            }
        }
        if(platform === 'mw_scss') {
            if(
                // less to sass conversion
                commit.message.match(/^\[MW-****\]/) ||
                // code styling
                commit.message.match(/^\[MW-****\]/) ||
                commit.message.match(/^\[MW-****\]/) ||
                commit.message.match(/^\[MW-****\]/) ||
                ...
            ) {
                return;
            }
        }
        if(platform === 'cosmos') {
            if(
                ...
            ) {
                return;
            }
        }

        const date = commit.date;
        let newFiles;
        let newInsertions;
        let newDeletions;
        const commitFiles = commit.files ? parseInt(commit.files, 10) : 0;
        const commitInsertions = commit.insertions ? parseInt(commit.insertions, 10) : 0;
        const commitDeletions = commit.deletions ? parseInt(commit.deletions, 10) : 0;

        if(aggregated[date]) {
            newFiles = aggregated[date]['files'] + commitFiles;
            newInsertions = aggregated[date]['insertions'] + commitInsertions;
            newDeletions = aggregated[date]['deletions'] + commitDeletions;
        } else {
            newFiles = commitFiles;
            newInsertions = commitInsertions;
            newDeletions = commitDeletions;
        }
        aggregated[date] = {
            date: date,
            files: newFiles,
            insertions: newInsertions,
            deletions: newDeletions,
        };
    });
    const aggregatedArr = [];
    let aggregatedTxt = 'date\tfiles\tinsertions\tdeletions\n';
    Object.keys(aggregated).forEach((date) => {
        const row = `${date}\t${aggregated[date].files}\t${aggregated[date].insertions}\t${aggregated[date].deletions}`;
        aggregatedTxt += `${row}\n`;
        aggregatedArr.push(aggregated[date]);
    });
    fse.writeFileSync(`./data-processed/aggregated-${platform}.tsv`, aggregatedTxt);
    fse.writeJsonSync(`./data-processed/aggregated-${platform}.json`, aggregatedArr);
    fse.writeJsonSync(`./application/src/data/aggregated-${platform}.json`, aggregatedArr);

    console.log(`Processing  git-log file for ${platform} completed.`);
});

A script to parse and process the git-log of a repository and output meaningful data to be visualized in a D3.js graph.

More details here: https://medium.com/@didoo/measuring-the-impact-of-a-design-system-7f925af090f7