A simple way to fetch Airtable API

AirtableConnector.ts

import * as path from "path";
import * as fs from "fs";
import axios, { AxiosError, type AxiosInstance } from "axios";

import airtableDefaults from "./constants/airtable-defaults";

const fsPromises = fs.promises;

const MAX_RETRIES = 3; // Number of times to retry after hitting rate limit
const RETRY_DELAY = 30 * 1000; // 30 seconds in milliseconds

export interface GetAllRecordsOptions {
  fields?: string[];
  filterByFormula?: string;
  sort?: { field: string; direction?: "asc" | "desc" }[];
  maxRecords?: number;
}

interface UnknownField {
  unknownField: string;
  defaultValue: string | number;
}

type Fields<T> = T extends undefined ? { [key: string]: any } : T;
export interface AirtableRecord<T> {
  id: string;
  createdTime: string;
  fields: Fields<T>;
}

interface AirtableFileRecordThumbnail {
  url: string;
  width: number;
  height: number;
}

export interface AirtableFileRecord {
  id: string;
  width: number;
  height: number;
  url: string;
  filename: string;
  size: number;
  type: string;
  thumbnails: {
    small: AirtableFileRecordThumbnail;
    large: AirtableFileRecordThumbnail;
    full: AirtableFileRecordThumbnail;
  };
}

class Record implements AirtableRecord<any> {
  id: string;
  createdTime: string;
  fields: Fields<any>;

  constructor(data: AirtableRecord<any>) {
    this.id = data.id;
    this.createdTime = data.createdTime;
    this.fields = data.fields;
  }

  get(fieldName: string): any {
    return this.fields[fieldName];
  }
}

const AIRTABLE_API_BASE_URL = "https://api.airtable.com/v0" as const;

class AirtableConnector {
  private axiosInstance: AxiosInstance;
  private unknownFields: UnknownField[];

  constructor(
    private airtableToken?: string,
    private baseId?: string,
  ) {
    this.unknownFields = [];

    if (!airtableToken) this.airtableToken = process.env.AIRTABLE_API_TOKEN;
    if (!baseId) this.baseId = process.env.AIRTABLE_BASE_ID;

    if (!this.airtableToken)
      throw new Error(
        "an Airtable token is required to use the AirtableConnector",
      );

    if (!this.baseId)
      throw new Error(
        "an Airtable base id is required to use the AirtableConnector",
      );

    this.axiosInstance = axios.create({
      baseURL: `${AIRTABLE_API_BASE_URL}/${this.baseId}`,
      headers: { Authorization: `Bearer ${this.airtableToken}` },
    });
  }

  private async requestWithRetry<T>(method: () => Promise<T>): Promise<T> {
    let retries = 0;

    while (retries < MAX_RETRIES) {
      try {
        return await method();
      } catch (error) {
        if (error instanceof AxiosError) {
          if (error.response && error.response.status === 429) {
            // Handle rate limit exceeded error
            console.warn(
              `Rate limit exceeded. Retrying in ${
                RETRY_DELAY / 1000
              } seconds...`,
            );
            await this.sleep(RETRY_DELAY);
          } else {
            throw error; // If it's another kind of error, throw it
          }
        } else {
          console.log("An error occurred:", error);
          throw error;
        }
        retries++; // Increment retries for all errors
      }
    }
    throw new Error(
      `Max retries (${MAX_RETRIES}) reached for Airtable API request`,
    );
  }

  private sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms));
  }

  async getSingleRecord(table: string, recordId: string): Promise<Record> {
    try {
      if (!table)
        throw new Error(
          "a table is required to use the getSingleRecord method",
        );

      if (!recordId) throw new Error("please pass in a valid recordId");

      const { data: record } = await this.requestWithRetry(() =>
        this.axiosInstance.get<AirtableRecord<any>>(`/${table}/${recordId}`),
      );
      return new Record(record);
    } catch (err) {
      console.log(err);
      throw err;
    }
  }

  async getAllRecords(
    table: string,
    options?: GetAllRecordsOptions,
  ): Promise<Record[]> {
    if (!table)
      throw new Error("a table is required to use the getAllRecords method");

    let hasOffset = true;
    let currentOffset = "";
    let totalRecords: Record[] = [];

    let loopNeedsToContinue: boolean = false;
    while (hasOffset || loopNeedsToContinue) {
      try {
        const params: GetAllRecordsOptions & {
          offset?: string;
          pageSize?: number;
        } = {
          pageSize: options?.maxRecords,
        };

        if (currentOffset) params.offset = currentOffset;
        if (options?.fields) {
          // Find the unknown field options we have encountered that have default values
          params.fields = [...options.fields].filter((field) => {
            return !this.unknownFields.some(
              (unknownField) => unknownField.unknownField === field,
            );
          });
        }
        if (options?.filterByFormula)
          params.filterByFormula = options.filterByFormula;
        if (options?.sort) params.sort = options.sort;

        const {
          data: { records, offset },
        }: { data: { records: AirtableRecord<any>[]; offset: string } } =
          await this.requestWithRetry(() =>
            this.axiosInstance.get(`/${table}`, { params }),
          );

        // Add to the records array all unknown fields with their default values
        const recordsWithDefaultValues = records.map((record) => {
          const updatedFields = { ...record.fields };
          this.unknownFields.forEach((unknownFieldObj) => {
            updatedFields[unknownFieldObj.unknownField] =
              unknownFieldObj.defaultValue;
          });
          return { ...record, fields: updatedFields };
        });

        const mappedRecords = recordsWithDefaultValues.map(
          (record): Record => new Record(record),
        );
        totalRecords = [...totalRecords, ...mappedRecords];

        if (offset) {
          currentOffset = offset;
        } else {
          currentOffset = "";
          hasOffset = false;
          loopNeedsToContinue = false;
        }
      } catch (err) {
        if (err instanceof AxiosError) {
          if (err.response?.data.error.type === "UNKNOWN_FIELD_NAME") {
            // Get the field name in the err.response.data.error.message string
            const unknownField: string = err.response.data.error.message
              .split('"')
              .at(1)
              .replace('"', "");

            // See in the airtable-defaults.ts if there's a default for this field
            const defaultValue = airtableDefaults.find(
              (item) => item.field === unknownField,
            );

            // throw if there's no value
            if (!defaultValue)
              throw new Error(
                `${err.response?.data.error.message} and astro-air could not find a default value.`,
              );

            // Add an option to the next loop iteration to remove the field from the params.
            this.unknownFields.push({
              unknownField,
              defaultValue: defaultValue.defaultValue,
            });

            // Make sure the loop continues
            loopNeedsToContinue = true;
          } else {
            const stringError = JSON.stringify(err.response?.data.error);
            throw new Error(stringError);
          }
        } else {
          // Handle rare unknown errors that are not AxiosErrors
          console.log(err);
          throw new Error("Unknown error");
        }
      }
    }
    return totalRecords;
  }

  async downloadAirtableFiles(
    fileArray: AirtableFileRecord[],
    options?: {
      targetFolder?: "public" | "assets";
    },
  ) {
    const targetFolder = options?.targetFolder;
    if (!fileArray.length) {
      console.log("You passed in:");
      console.log(fileArray);
      throw new Error("Please pass in an array of Airtable files");
    }
    if (!fileArray.every((item) => "url" in item))
      throw new Error("Please pass in a valid Airtable file item");

    const uploadedFiles: string[] = [];

    const staticFolderPath = path.join(
      process.cwd(),
      targetFolder && targetFolder === "assets" ? "src/assets" : "public",
    );
    if (!fs.existsSync(staticFolderPath)) {
      await fsPromises.mkdir(staticFolderPath);
    }
    for (let file of fileArray) {
      const fileUrl = file.url;
      const filePath = path.join(staticFolderPath, file.filename);

      // Download file from file.url using axios
      const fileResponse = await axios({
        url: fileUrl,
        method: "GET",
        responseType: "stream",
      });

      // Write file to static folder
      await fsPromises.writeFile(filePath, fileResponse.data);
      uploadedFiles.push(file.filename);
    }

    return uploadedFiles;
  }
}

export default AirtableConnector;