Fix package by testing real world examples
This commit is contained in:
Chris Daßler
548
bin/NodeImportLoader.js
Normal file
548
bin/NodeImportLoader.js
Normal file
@@ -0,0 +1,548 @@
|
||||
import { promises as fs } from 'node:fs';
|
||||
import { register } from 'node:module';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { ModuleKind, ScriptTarget, transpileModule } from 'typescript';
|
||||
// Import shared core functions - will be loaded dynamically
|
||||
let iorCore = null;
|
||||
// Dynamic import of core functions
|
||||
const loadIorCore = async () => {
|
||||
if (!iorCore) {
|
||||
iorCore = await import('../lib/ior-core.js');
|
||||
}
|
||||
return iorCore;
|
||||
};
|
||||
register('./src/loaders/NodeImportLoader.ts', pathToFileURL('./'));
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const projectRoot = process.cwd(); // Use current working directory since this is a bin script
|
||||
const DEBUG = process.env.DEBUG === 'true';
|
||||
// Create a simple console logger to avoid circular dependency issues
|
||||
const logger = {
|
||||
debug: (...args) => DEBUG && console.log('[NodeImportLoader:DEBUG]', ...args),
|
||||
info: (...args) => console.log('[NodeImportLoader:INFO]', ...args),
|
||||
warn: (...args) => console.warn('[NodeImportLoader:WARN]', ...args),
|
||||
error: (...args) => console.error('[NodeImportLoader:ERROR]', ...args),
|
||||
};
|
||||
const componentCache = new Map();
|
||||
const remoteCache = new Map();
|
||||
const typeCache = new Map();
|
||||
const processedTypes = new Set();
|
||||
/**
|
||||
* Fetch type definitions for a component
|
||||
*/
|
||||
async function fetchTypeDefinitions(config, componentName, version) {
|
||||
const baseUrl = config.type === 'github'
|
||||
? `https://raw.githubusercontent.com/${config.owner}/${config.repository}/${config.branch || 'main'}`
|
||||
: config.type === 'gitea'
|
||||
? `https://${config.instance}/${config.owner}/${config.repository}/raw/branch/${config.branch || 'main'}`
|
||||
: null;
|
||||
if (!baseUrl)
|
||||
return null;
|
||||
const componentPath = `src/components/${componentName}/${version}`;
|
||||
const possiblePaths = [
|
||||
`${componentPath}/${componentName}.d.ts`,
|
||||
`${componentPath}/index.d.ts`,
|
||||
`${componentPath}/types.d.ts`,
|
||||
];
|
||||
for (const typePath of possiblePaths) {
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/${typePath}`);
|
||||
if (response.ok) {
|
||||
return await response.text();
|
||||
}
|
||||
}
|
||||
catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Generate and save type declarations
|
||||
*/
|
||||
async function generateTypeDeclaration(ior, isRemote = false) {
|
||||
// Avoid duplicate processing
|
||||
const cacheKey = `${isRemote ? 'remote:' : 'local:'}${ior}`;
|
||||
if (processedTypes.has(cacheKey)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
let typeContent = null;
|
||||
if (isRemote) {
|
||||
// For remote components, fetch type definitions
|
||||
const core = await loadIorCore();
|
||||
const config = core.parseRemoteIOR(`ior:${ior}`);
|
||||
if (config) {
|
||||
const parts = ior.split(':');
|
||||
const componentAndVersion = parts[parts.length - 1];
|
||||
const [componentName, version] = componentAndVersion.split('@');
|
||||
typeContent = await fetchTypeDefinitions(config, componentName, version);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// For local components, read the .d.ts file
|
||||
const atIndex = ior.lastIndexOf('@');
|
||||
const componentFullName = ior.substring(0, atIndex);
|
||||
const version = ior.substring(atIndex + 1);
|
||||
const componentName = componentFullName.split('.').pop();
|
||||
if (!componentName) {
|
||||
throw new Error(`Invalid IOR format: ${ior}`);
|
||||
}
|
||||
const componentsDir = path.join(projectRoot, 'src', 'components');
|
||||
const possiblePaths = [
|
||||
path.join(componentsDir, componentName, version, `${componentName}.d.ts`),
|
||||
path.join(componentsDir, componentName, version, 'index.d.ts'),
|
||||
path.join(componentsDir, componentName, version, 'types.d.ts'),
|
||||
];
|
||||
for (const dtsPath of possiblePaths) {
|
||||
try {
|
||||
typeContent = await fs.readFile(dtsPath, 'utf-8');
|
||||
break;
|
||||
}
|
||||
catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (typeContent) {
|
||||
// Check if the type content already contains the correct module declaration
|
||||
const moduleDeclaration = `ior:${isRemote ? '' : 'esm:'}${ior}`;
|
||||
// Only process if the content doesn't already have multiple module declarations
|
||||
// or if it doesn't contain the target module declaration
|
||||
if (typeContent.includes(`declare module "${moduleDeclaration}"`) ||
|
||||
typeContent.includes(`declare module '${moduleDeclaration}'`)) {
|
||||
// Content already has the correct module declaration, use as-is
|
||||
// This preserves any additional module declarations (like 'hypercore')
|
||||
}
|
||||
else if (!typeContent.includes('declare module')) {
|
||||
// No module declaration at all, wrap the content
|
||||
typeContent = `declare module "${moduleDeclaration}" {\n${typeContent}\n}\n`;
|
||||
}
|
||||
// If it has other module declarations but not our target, leave it as-is
|
||||
// This preserves files with multiple module declarations
|
||||
// Save to type cache
|
||||
typeCache.set(cacheKey, {
|
||||
content: typeContent,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
// Write to global types file
|
||||
await updateGlobalTypes();
|
||||
}
|
||||
processedTypes.add(cacheKey);
|
||||
}
|
||||
catch (err) {
|
||||
DEBUG && logger.debug(`Failed to generate types for ${ior}:`, err);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update global types file with all cached types
|
||||
*/
|
||||
async function updateGlobalTypes() {
|
||||
const typesDir = path.join(projectRoot, 'src', 'types');
|
||||
const globalTypesPath = path.join(typesDir, 'auto-generated.d.ts');
|
||||
let content = '// Auto-generated type declarations by NodeImportLoader\n';
|
||||
content += `// Generated at: ${new Date().toISOString()}\n`;
|
||||
content += '// DO NOT EDIT - This file is automatically maintained\n\n';
|
||||
// Add all cached type definitions
|
||||
for (const [key, entry] of typeCache.entries()) {
|
||||
content += `// ${key}\n`;
|
||||
content += entry.content;
|
||||
content += '\n\n';
|
||||
}
|
||||
// Ensure directory exists
|
||||
await fs.mkdir(typesDir, { recursive: true });
|
||||
// Write the file
|
||||
await fs.writeFile(globalTypesPath, content, 'utf-8');
|
||||
DEBUG && logger.debug(`Updated global types: ${globalTypesPath}`);
|
||||
}
|
||||
/**
|
||||
* Cache remote component locally
|
||||
*/
|
||||
async function cacheRemoteComponent(ior, content, ttl = 3600000) {
|
||||
const core = await loadIorCore();
|
||||
const hash = core.generateCacheHash(ior);
|
||||
const cacheDir = path.join(tmpdir(), 'metatrom-components', 'remote');
|
||||
const cachePath = path.join(cacheDir, `${hash}.ts`);
|
||||
// Create cache directory if it doesn't exist
|
||||
await fs.mkdir(cacheDir, { recursive: true });
|
||||
// Write content to cache
|
||||
await fs.writeFile(cachePath, content, 'utf-8');
|
||||
// Update cache map
|
||||
remoteCache.set(ior, {
|
||||
path: cachePath,
|
||||
timestamp: Date.now(),
|
||||
ttl,
|
||||
});
|
||||
DEBUG && logger.debug(`Cached remote component: ${ior} -> ${cachePath}`);
|
||||
return cachePath;
|
||||
}
|
||||
/**
|
||||
* Get cached remote component if valid
|
||||
*/
|
||||
async function getCachedRemoteComponent(ior) {
|
||||
const entry = remoteCache.get(ior);
|
||||
if (!entry) {
|
||||
return null;
|
||||
}
|
||||
// Check if cache entry is still valid
|
||||
if (Date.now() - entry.timestamp > entry.ttl) {
|
||||
remoteCache.delete(ior);
|
||||
// Try to delete the file (non-blocking)
|
||||
fs.unlink(entry.path).catch(() => {
|
||||
logger.error(`Error deleting cached file ${entry.path}`);
|
||||
});
|
||||
return null;
|
||||
}
|
||||
// Verify file still exists
|
||||
try {
|
||||
await fs.access(entry.path);
|
||||
DEBUG && logger.debug(`Using cached component: ${ior}`);
|
||||
return entry.path;
|
||||
}
|
||||
catch {
|
||||
remoteCache.delete(ior);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Fetch remote component based on IOR
|
||||
*/
|
||||
async function fetchRemoteComponent(ior) {
|
||||
// Check cache first
|
||||
const cached = await getCachedRemoteComponent(ior);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
const core = await loadIorCore();
|
||||
const config = core.parseRemoteIOR(ior);
|
||||
if (!config) {
|
||||
throw new Error(`Invalid remote IOR format: ${ior}`);
|
||||
}
|
||||
// Extract component name and version from IOR
|
||||
const parts = ior.split(':');
|
||||
const componentAndVersion = parts[parts.length - 1];
|
||||
const [componentName, version] = componentAndVersion.split('@');
|
||||
if (!componentName || !version) {
|
||||
throw new Error(`Missing component name or version in IOR: ${ior}`);
|
||||
}
|
||||
let content;
|
||||
switch (config.type) {
|
||||
case 'github':
|
||||
const githubResult = await core.fetchFromGitHub(config);
|
||||
// For Node.js, we need to extract the content differently since we're dealing with archives
|
||||
// For now, fetch just the main file directly (simplified approach)
|
||||
content = await fetchComponentFile(config, componentName, version);
|
||||
break;
|
||||
case 'gitea':
|
||||
const giteaResult = await core.fetchFromGitea(config);
|
||||
// Similarly for Gitea
|
||||
content = await fetchComponentFile(config, componentName, version);
|
||||
break;
|
||||
case 'ipfs':
|
||||
const ipfsResult = await core.fetchFromIPFS(config);
|
||||
content = ipfsResult.content;
|
||||
break;
|
||||
case 'p2p':
|
||||
throw new Error(`P2P fetching not yet implemented. PeerID: ${config.peerId}`);
|
||||
default:
|
||||
throw new Error(`Unsupported remote type: ${config.type}`);
|
||||
}
|
||||
// Cache the fetched component
|
||||
const cachedPath = await cacheRemoteComponent(ior, content);
|
||||
return cachedPath;
|
||||
}
|
||||
/**
|
||||
* Fetch component file directly (simplified approach for Node.js)
|
||||
*/
|
||||
async function fetchComponentFile(config, componentName, version) {
|
||||
let baseUrl;
|
||||
if (config.type === 'github') {
|
||||
baseUrl = `https://raw.githubusercontent.com/${config.owner}/${config.repository}/main`;
|
||||
}
|
||||
else if (config.type === 'gitea') {
|
||||
baseUrl = `https://${config.instance}/${config.owner}/${config.repository}/raw/branch/main`;
|
||||
}
|
||||
else {
|
||||
throw new Error(`Unsupported type for direct fetch: ${config.type}`);
|
||||
}
|
||||
const componentPath = `src/components/${componentName}/${version}`;
|
||||
// Try to fetch the main component file
|
||||
const possibleFiles = [
|
||||
`${componentPath}/index.ts`,
|
||||
`${componentPath}/${componentName}.ts`,
|
||||
`${componentPath}/index.tsx`,
|
||||
`${componentPath}/${componentName}.tsx`,
|
||||
];
|
||||
for (const file of possibleFiles) {
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/${file}`);
|
||||
if (response.ok) {
|
||||
return await response.text();
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Try next file
|
||||
}
|
||||
}
|
||||
throw new Error(`Could not fetch component file for ${componentName}@${version}`);
|
||||
}
|
||||
/**
|
||||
* Get component path for local components using package.json
|
||||
*/
|
||||
async function getLocalComponentPath(ior) {
|
||||
if (componentCache.has(ior))
|
||||
return componentCache.get(ior);
|
||||
// IOR format: com.metatrom.examples.componentname@version
|
||||
const atIndex = ior.lastIndexOf('@');
|
||||
if (atIndex === -1) {
|
||||
throw new Error(`Invalid IOR format (missing @): ${ior}`);
|
||||
}
|
||||
const componentFullName = ior.substring(0, atIndex);
|
||||
const version = ior.substring(atIndex + 1);
|
||||
if (!componentFullName || !version) {
|
||||
throw new Error(`Invalid IOR format: ${ior}`);
|
||||
}
|
||||
// Search for components with matching package.json
|
||||
const componentsDir = path.join(projectRoot, 'src', 'components');
|
||||
try {
|
||||
const componentDirs = await fs.readdir(componentsDir, {
|
||||
withFileTypes: true,
|
||||
});
|
||||
for (const dir of componentDirs) {
|
||||
if (dir.isDirectory()) {
|
||||
const versionDir = path.join(componentsDir, dir.name, version);
|
||||
const packagePath = path.join(versionDir, 'package.json');
|
||||
try {
|
||||
const core = await loadIorCore();
|
||||
const pkg = await core.parsePackageJsonAsync(packagePath);
|
||||
if (pkg && pkg.metatrom?.ior === ior) {
|
||||
// Found matching component - prefer index.ts if it exists
|
||||
const indexPath = path.join(versionDir, 'index.ts');
|
||||
try {
|
||||
await fs.access(indexPath);
|
||||
DEBUG && logger.debug(`Mapped IOR ${ior} to ${indexPath}`);
|
||||
componentCache.set(ior, indexPath);
|
||||
return indexPath;
|
||||
}
|
||||
catch {
|
||||
// Fall back to main file specified in package.json
|
||||
const mainFile = pkg.main || `${dir.name}.ts`;
|
||||
const tsPath = path.join(versionDir, mainFile);
|
||||
await fs.access(tsPath);
|
||||
DEBUG && logger.debug(`Mapped IOR ${ior} to ${tsPath}`);
|
||||
componentCache.set(ior, tsPath);
|
||||
return tsPath;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
DEBUG &&
|
||||
logger.debug(`Skipping ${packagePath}:`, error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
throw new Error(`Failed to read components directory: ${err}`);
|
||||
}
|
||||
throw new Error(`No component found for IOR ${ior}`);
|
||||
}
|
||||
/**
|
||||
* Main resolve function for Node.js module resolution
|
||||
*/
|
||||
export async function resolve(specifier, context, defaultResolve) {
|
||||
DEBUG &&
|
||||
logger.debug(`Resolving specifier: ${specifier}, parentURL: ${context.parentURL}`);
|
||||
if (specifier.startsWith('ior:')) {
|
||||
// Check if it's a remote IOR
|
||||
const core = await loadIorCore();
|
||||
const remoteConfig = core.parseRemoteIOR(specifier);
|
||||
if (remoteConfig) {
|
||||
// Handle remote component
|
||||
try {
|
||||
const tsPath = await fetchRemoteComponent(specifier);
|
||||
// Generate type definitions for remote component
|
||||
const iorPart = specifier.substring(4); // Remove "ior:"
|
||||
await generateTypeDeclaration(iorPart, true);
|
||||
DEBUG && logger.debug(`Resolved remote IOR ${specifier} to ${tsPath}`);
|
||||
return {
|
||||
url: pathToFileURL(tsPath).href,
|
||||
format: 'module',
|
||||
shortCircuit: true,
|
||||
type: 'typescript',
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error(`Failed to fetch remote component ${specifier}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Handle local component with format: ior:esm:com.metatrom.examples.component@version
|
||||
// Remove the "ior:" prefix and extract the IOR
|
||||
const iorWithPrefix = specifier.substring(4); // Remove "ior:"
|
||||
// Check if it has "esm:" prefix
|
||||
let actualIor = iorWithPrefix;
|
||||
if (iorWithPrefix.startsWith('esm:')) {
|
||||
actualIor = iorWithPrefix.substring(4); // Remove "esm:"
|
||||
}
|
||||
const tsPath = await getLocalComponentPath(actualIor);
|
||||
// Generate type definitions for local component
|
||||
await generateTypeDeclaration(actualIor, false);
|
||||
DEBUG && logger.debug(`Resolving local IOR ${actualIor} to ${tsPath}`);
|
||||
return {
|
||||
url: pathToFileURL(tsPath).href,
|
||||
format: 'module',
|
||||
shortCircuit: true,
|
||||
type: 'typescript',
|
||||
};
|
||||
}
|
||||
}
|
||||
// Handle relative .ts imports
|
||||
if (specifier.startsWith('./') || specifier.startsWith('../')) {
|
||||
const parentURL = context.parentURL
|
||||
? fileURLToPath(context.parentURL)
|
||||
: __dirname;
|
||||
const baseDir = path.dirname(parentURL);
|
||||
// Try with .ts extension first
|
||||
const tsPath = path.resolve(baseDir, `${specifier}.ts`);
|
||||
try {
|
||||
await fs.access(tsPath);
|
||||
DEBUG && logger.debug(`Resolving relative .ts file: ${tsPath}`);
|
||||
return {
|
||||
url: pathToFileURL(tsPath).href,
|
||||
format: 'module',
|
||||
shortCircuit: true,
|
||||
type: 'typescript',
|
||||
};
|
||||
}
|
||||
catch {
|
||||
// Try without adding extension (file might already have .ts)
|
||||
const directPath = path.resolve(baseDir, specifier);
|
||||
try {
|
||||
await fs.access(directPath);
|
||||
if (directPath.endsWith('.ts')) {
|
||||
DEBUG && logger.debug(`Resolving relative .ts file: ${directPath}`);
|
||||
return {
|
||||
url: pathToFileURL(directPath).href,
|
||||
format: 'module',
|
||||
shortCircuit: true,
|
||||
type: 'typescript',
|
||||
};
|
||||
}
|
||||
}
|
||||
catch {
|
||||
DEBUG &&
|
||||
logger.debug(`No .ts file found for ${specifier}, falling back to default resolver`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return defaultResolve(specifier, context, defaultResolve);
|
||||
}
|
||||
/**
|
||||
* Load and transpile TypeScript files
|
||||
*/
|
||||
export async function load(url, context, defaultLoad) {
|
||||
if (url.endsWith('.ts')) {
|
||||
const filePath = fileURLToPath(url);
|
||||
DEBUG && logger.debug(`Loading TypeScript file: ${filePath}`);
|
||||
try {
|
||||
const code = await fs.readFile(filePath, 'utf-8');
|
||||
const transpiled = transpileModule(code, {
|
||||
compilerOptions: {
|
||||
module: ModuleKind.ESNext,
|
||||
target: ScriptTarget.ESNext,
|
||||
noEmitHelpers: true,
|
||||
importHelpers: false,
|
||||
strict: true,
|
||||
sourceMap: true,
|
||||
},
|
||||
});
|
||||
return {
|
||||
format: 'module',
|
||||
source: transpiled.outputText,
|
||||
shortCircuit: true,
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error(`Failed to transpile ${filePath}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return defaultLoad(url, context, defaultLoad);
|
||||
}
|
||||
/**
|
||||
* Clear cache for a specific IOR or all cached components
|
||||
*/
|
||||
export function clearCache(ior) {
|
||||
if (ior) {
|
||||
componentCache.delete(ior);
|
||||
remoteCache.delete(ior);
|
||||
}
|
||||
else {
|
||||
componentCache.clear();
|
||||
remoteCache.clear();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Set cache TTL for remote components
|
||||
*/
|
||||
export function setCacheTTL(ttl) {
|
||||
// This could be made more sophisticated with per-source TTL
|
||||
for (const entry of remoteCache.values()) {
|
||||
entry.ttl = ttl;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Preload remote components for faster access
|
||||
*/
|
||||
export async function preloadComponent(ior) {
|
||||
try {
|
||||
await fetchRemoteComponent(ior);
|
||||
DEBUG && logger.debug(`Preloaded component: ${ior}`);
|
||||
}
|
||||
catch (error) {
|
||||
logger.error(`Failed to preload component ${ior}:`, error);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Initialize type generation for all local components
|
||||
*/
|
||||
async function initializeLocalTypes() {
|
||||
const componentsDir = path.join(projectRoot, 'src', 'components');
|
||||
try {
|
||||
const dirs = await fs.readdir(componentsDir, { withFileTypes: true });
|
||||
for (const dir of dirs) {
|
||||
if (dir.isDirectory()) {
|
||||
const componentDir = path.join(componentsDir, dir.name);
|
||||
const versions = await fs.readdir(componentDir, {
|
||||
withFileTypes: true,
|
||||
});
|
||||
for (const version of versions) {
|
||||
if (version.isDirectory()) {
|
||||
const packagePath = path.join(componentDir, version.name, 'package.json');
|
||||
try {
|
||||
const packageContent = await fs.readFile(packagePath, 'utf-8');
|
||||
const pkg = JSON.parse(packageContent);
|
||||
if (pkg.metatrom?.ior) {
|
||||
await generateTypeDeclaration(pkg.metatrom.ior, false);
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Skip if no package.json or invalid
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
DEBUG && logger.debug('Initialized type definitions for local components');
|
||||
}
|
||||
catch (error) {
|
||||
DEBUG && logger.debug('Failed to initialize local types:', error);
|
||||
}
|
||||
}
|
||||
// Auto-initialize types on first import
|
||||
(async () => {
|
||||
await initializeLocalTypes();
|
||||
})();
|
||||
Reference in New Issue
Block a user