Offer script to help to collect variables from submodules to main one

[Hi-Fi]

Description

When having multiple submodules in the project, copying and managing variables can be quite tedious thing.

Collection could be e.g. done so that submodules have variables defined in TS, and then main module allows to pass those through. This also keeps registry UI nicer as by default there's no main module (example: https://registry.terraform.io/modules/Hi-Fi/serverless-github-actions-runner-controller/cdk/0.0.0)

Example code from Gemini:

import * as fs from 'fs';
import * as path from 'path';
import {
  extractVariables,
  findSubmoduleVariables,
  generateRootVariablesTf,
  VariableDefinition,
  SubmoduleVariables,
} from './collect-vars'; // Import the functions from your script

// Mock file system functions for testing
jest.mock('fs', () => ({
  existsSync: jest.fn(),
  readFileSync: jest.fn(),
  readdirSync: jest.fn(),
  writeFileSync: jest.fn(),
  statSync: jest.fn(),
}));

// Helper function to create a mock file structure
const createMockFileSystem = (files: Record<string, string>) => {
  const mockExistsSync = fs.existsSync as jest.MockedFunction<typeof fs.existsSync>;
  const mockReadFileSync = fs.readFileSync as jest.MockedFunction<typeof fs.readFileSync>;
  const mockReaddirSync = fs.readdirSync as jest.MockedFunction<typeof fs.readdirSync>;
  const mockStatSync = fs.statSync as jest.MockedFunction<typeof fs.statSync>;

  mockExistsSync.mockImplementation((filePath: string) => {
    return files.hasOwnProperty(filePath);
  });

  mockReadFileSync.mockImplementation((filePath: string, encoding: string) => {
    if (files.hasOwnProperty(filePath)) {
      return files[filePath];
    }
    throw new Error(`File not found: ${filePath}`);
  });

  mockReaddirSync.mockImplementation((dirPath: string) => {
    const fileList: string[] = [];
    for (const filePath in files) {
      if (filePath.startsWith(dirPath)) {
        const relativePath = path.relative(dirPath, filePath);
        const parts = relativePath.split(path.sep);
        if (parts.length > 0) {
          const firstPart = parts[0];
          if (!fileList.includes(firstPart)) { // Avoid duplicates
             fileList.push(firstPart);
          }
        }
      }
    }
    return fileList;
  });

  mockStatSync.mockImplementation((filePath: string) => {
    if (files.hasOwnProperty(filePath)) {
      const isDirectory = !filePath.endsWith('.json') && !filePath.endsWith('.tf'); // simplistic check
      return {
        isDirectory: () => isDirectory,
        isFile: () => !isDirectory,
      } as fs.Stats;
    }
    throw new Error(`File not found: ${filePath}`);
  });
};

describe('extractVariables', () => {
  it('should extract variables from a valid cdk.tf.json file', () => {
    const filePath = 'test.cdk.tf.json';
    const fileContent = `{
      "variable": {
        "my_var": {
          "description": "My variable description"
        },
        "another_var": {
          "description": "Another description",
          "customInfo": {
            "key1": "value1",
            "key2": 123
          }
        }
      }
    }`;
    createMockFileSystem({ [filePath]: fileContent });

    const result = extractVariables(filePath);
    expect(result).toEqual([
      {
        name: 'my_var',
        description: 'My variable description',
        submodules: [],
        customInfo: {},
      },
      {
        name: 'another_var',
        description: 'Another description',
        submodules: [],
        customInfo: {
          key1: 'value1',
          key2: 123,
        },
      },
    ]);
  });

  it('should handle a cdk.tf.json file with no variables', () => {
    const filePath = 'test.cdk.tf.json';
    const fileContent = `{}`;
    createMockFileSystem({ [filePath]: fileContent });

    const result = extractVariables(filePath);
    expect(result).toEqual([]);
  });

  it('should handle a cdk.tf.json file with an empty variable object', () => {
    const filePath = 'test.cdk.tf.json';
    const fileContent = `{ "variable": {} }`;
    createMockFileSystem({ [filePath]: fileContent });

    const result = extractVariables(filePath);
    expect(result).toEqual([]);
  });

    it('should handle missing description', () => {
    const filePath = 'test.cdk.tf.json';
    const fileContent = `{
      "variable": {
        "my_var": {
        }
      }
    }`;
    createMockFileSystem({ [filePath]: fileContent });

    const result = extractVariables(filePath);
    expect(result).toEqual([
      {
        name: 'my_var',
        description: 'No description provided.',
        submodules: [],
        customInfo: {},
      },
    ]);
  });

  it('should handle invalid JSON', () => {
    const filePath = 'test.cdk.tf.json';
    const fileContent = `invalid json`;
    createMockFileSystem({ [filePath]: fileContent });

    const result = extractVariables(filePath);
    expect(result).toEqual([]);
  });
});

describe('findSubmoduleVariables', () => {
  it('should find variables in nested submodules within the modules directory', () => {
    const mockFiles = {
      'modules/subdir1/subdir2/test.cdk.tf.json': `{
        "variable": {
          "nested_var": {
            "description": "Nested variable"
          }
        }
      }`,
      'modules/another_module/test.cdk.tf.json': `{
        "variable": {
          "another_var": {
            "description": "Another variable"
          }
        }
      }`,
      'root_file.tf': 'some content', // should be ignored
      'modules/empty_dir/empty.tf': ''
    };
    createMockFileSystem(mockFiles);

    const result = findSubmoduleVariables('.');
    expect(result).toEqual({
      'subdir1/subdir2': [
        {
          name: 'nested_var',
          description: 'Nested variable',
          submodules: [],
          customInfo: {},
        },
      ],
      'another_module': [
        {
          name: 'another_var',
          description: 'Another variable',
          submodules: [],
          customInfo: {},
        },
      ],
    });
  });

  it('should return an empty object if no modules directory exists', () => {
    createMockFileSystem({}); // No files at all
    const result = findSubmoduleVariables('.');
    expect(result).toEqual({});
  });

    it('should handle empty modules directory', () => {
        const mockFiles = {
            'modules/.keep': ''
        };
        createMockFileSystem(mockFiles);
        const result = findSubmoduleVariables('.');
        expect(result).toEqual({});
    });
});

describe('generateRootVariablesTf', () => {
  it('should generate variables.tf content with combined variables', () => {
    const collectedVariables: SubmoduleVariables = {
      'module1': [
        { name: 'common_var', description: 'Common var desc 1', submodules: [], customInfo: {  } },
        { name: 'module1_var', description: 'Module 1 var', submodules: [], customInfo: {} },
      ],
      'module2': [
        { name: 'common_var', description: 'Common var desc 2', submodules: [], customInfo: { a: 1 } },
        { name: 'module2_var', description: 'Module 2 var', submodules: [], customInfo: {} },
      ],
    };
    const mockWriteFileSync = fs.writeFileSync as jest.MockedFunction<typeof fs.writeFileSync>;
    mockWriteFileSync.mockImplementation(() => {});

    generateRootVariablesTf(collectedVariables);

    const expectedOutput = `# Combined variables from submodules

variable "common_var" {
  type = any
  description = "Common var desc 1 (Defined in: modules/module1, modules/module2)"
}

variable "module1_var" {
  type = any
  description = "Module 1 var (Defined in: modules/module1)"
}

variable "module2_var" {
  type = any
  description = "Module 2 var (Defined in: modules/module2)"
}

# Variables generated by terraform-variable-collector end
`;
    expect(mockWriteFileSync).toHaveBeenCalledWith('variables.tf', expectedOutput);
  });

  it('should handle empty collected variables', () => {
    const collectedVariables: SubmoduleVariables = {};
    const mockWriteFileSync = fs.writeFileSync as jest.MockedFunction<typeof fs.writeFileSync>;
    mockWriteFileSync.mockImplementation(() => {});

    generateRootVariablesTf(collectedVariables);

    const expectedOutput = `# Variables generated by terraform-variable-collector end
`;
    expect(mockWriteFileSync).toHaveBeenCalledWith('variables.tf', expectedOutput);
  });

  it('should preserve existing content in variables.tf', () => {
    const existingContent = `
# This is some existing content
terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 4.0"
    }
  }
}

variable "existing_var" {
  type = string
  description = "An existing variable"
}
`;
    const collectedVariables: SubmoduleVariables = {
      'module1': [
        { name: 'new_var', description: 'A new variable', submodules: [], customInfo: {} },
      ],
    };
    const mockExistsSync = fs.existsSync as jest.MockedFunction<typeof fs.existsSync>;
    const mockReadFileSync = fs.readFileSync as jest.MockedFunction<typeof fs.readFileSync>;
    const mockWriteFileSync = fs.writeFileSync as jest.MockedFunction<typeof fs.writeFileSync>;

    mockExistsSync.mockReturnValue(true);
    mockReadFileSync.mockReturnValue(existingContent);
    mockWriteFileSync.mockImplementation(() => {});

    generateRootVariablesTf(collectedVariables);

    const expectedOutput = `
# This is some existing content
terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 4.0"
    }
  }
}

variable "existing_var" {
  type = string
  description = "An existing variable"
}
# Variables generated by terraform-variable-collector start
variable "new_var" {
  type = any
  description = "A new variable (Defined in: modules/module1)"
}

# Variables generated by terraform-variable-collector end
`;
    expect(mockWriteFileSync).toHaveBeenCalledWith('variables.tf', expectedOutput);
  });
});

References

No response

Help Wanted

• I'm interested in contributing a fix myself

Community Note

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment