V8词法分析

lexer.cpp

#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <vector>

#include "src/v8.h"
#include "src/string-stream.h"
#include "src/scanner.h"
#include "src/scanner-character-streams.h"

using namespace v8::internal;

const byte* ReadFile(const char* name, int* length) {
  FILE* file = fopen(name, "rb");
  *length = 0;
  if (file == NULL) return NULL;

  fseek(file, 0, SEEK_END);
  int file_size = ftell(file);
  *length = file_size;
  rewind(file);

  byte* chars = new byte[file_size + 1];
  for (int i = 0; i < file_size;) {
    int read = static_cast<int>(fread(&chars[i], 1, file_size - i, file));
    i += read;
  }
  fclose(file);

  chars[file_size] = 0;

  return chars;
}

char* substr(const byte* source, int start, int end) {
  char* newstr = new char[end - start + 1];
  for (int i = start; i < end; i++) {
    newstr[i - start] = source[i];
  }
  newstr[end - start] = 0;
  return newstr;
}

struct TokenWithLocation {
  Token::Value value;
  size_t beg;
  size_t end;
  TokenWithLocation() : value(Token::ILLEGAL), beg(0), end(0) { }
  TokenWithLocation(Token::Value value, size_t beg, size_t end) :
      value(value), beg(beg), end(end) { }
  bool operator==(const TokenWithLocation& other) {
    return value == other.value && beg == other.beg && end == other.end;
  }
  bool operator!=(const TokenWithLocation& other) {
    return !(*this == other);
  }
  void Print(const char* prefix, const byte* content) const {
    printf("%s %11s at (%d, %d) \t: %s\n",
           prefix, Token::Name(value),
           static_cast<int>(beg),
           static_cast<int>(end),
           substr(content, beg, end));
  }
};

void PrintTokens(const char* name,
                 const std::vector<TokenWithLocation>& tokens,
                 const byte* content) {
  printf("No of tokens: %d\n",
         static_cast<int>(tokens.size()));
  printf("%s:\n", name);
  for (size_t i = 0; i < tokens.size(); ++i) {
    tokens[i].Print("=>", content);
  }
}

Token::Value Next(Scanner* scanner_, int* beg_pos, int* end_pos) {
  Token::Value res = scanner_->Next();
  *beg_pos = scanner_->location().beg_pos;
  *end_pos = scanner_->location().end_pos;
  return res;
}

int main(int argc, char* argv[]) {
  if (argc < 0) {
    printf("Please use it like: lexer filename.js");
    return 0;
  }
  char* filename = argv[1];

  v8::V8::Initialize();
  v8::Isolate* isolate = v8::Isolate::New();
  {
    v8::Isolate::Scope isolate_scope(isolate);
    v8::HandleScope handle_scope(isolate);
    v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New(isolate);
    v8::Local<v8::Context> context = v8::Context::New(isolate, NULL, global);
    {
      v8::Context::Scope scope(context);
      int length = 0;
      const byte* content = ReadFile(filename, &length);
      printf("%s\n", content);
      UnicodeCache* unicode_cache_ = new UnicodeCache();
      Scanner* scanner_ = new Scanner(unicode_cache_);
      BufferedUtf16CharacterStream* stream_ = new Utf8ToUtf16CharacterStream(content, length);
      scanner_->Initialize(stream_);
      std::vector<TokenWithLocation> tokens;
      Token::Value token;
      int beg, end;
      do {
        token = Next(scanner_, &beg, &end);
        tokens.push_back(TokenWithLocation(token, beg, end));
      } while (token != Token::EOS);
      PrintTokens("Baseline", tokens, content);
    }
  }
  v8::V8::Dispose();
  return 0;
}

Makefile

V8HOME = /Users/jacksontian/git/v8
FLAGS = -I$(V8HOME) \
	$(V8HOME)/out/x64.release/libv8_{libbase,base,snapshot}.a \
	$(V8HOME)/out/x64.release/libicu{uc,i18n,data}.a -stdlib=libstdc++

lexer:
	clang++ lexer.cpp -o bin/lexer $(FLAGS)