JsAmbientCGLoader.js

 
// Use global fetch if available (Node.js v18+), otherwise use node-fetch
const fetch =
    typeof globalThis.fetch === 'function'
        ? globalThis.fetch
        : (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args));
const fs = require('fs');
const { parse } = require('csv-parse/sync');
 
class AmbientCGLoader {
    constructor() {
        if (AmbientCGLoader.instance) {
            return AmbientCGLoader.instance;
        }
 
        this.logger = console;
        this.database = {};
        this.materials = null;
        this.materialNames = [];
        this.csvMaterials = null;
        this.downloadMaterial = null;
        this.downloadMaterialFileName = '';
 
        // Cache the instance
        AmbientCGLoader.instance = this;
    }
 
    loadMaterialsFromCache() {
        const MATERIALS_CACHE_FILE = 'ambientcg_materials.json';
        if (fs.existsSync(MATERIALS_CACHE_FILE)) {
            try {
                const data = fs.readFileSync(MATERIALS_CACHE_FILE, 'utf8');
                this.materials = JSON.parse(data);
                this.logger.info(`Loaded AmbientCG materials from cache: ${MATERIALS_CACHE_FILE}`);
            } catch (e) {
                this.logger.warn(`Failed to load AmbientCG materials cache: ${e.message}`);
            }
        }
    }
 
    setDebugging(debug = true) {
        this.logger.level = debug ? 'debug' : 'info';
    }
 
    getMaterialNames(key = 'assetId') {
        this.materialNames = [];
        const uniqueNames = new Set();
        if (this.materials) {
            this.materials.forEach(item => uniqueNames.add(item[key]));
        }
        this.materialNames = Array.from(uniqueNames).sort();
        return this.materialNames;
    }
 
    writeMaterialList(materialList, filename) {
        this.logger.info(`Writing material list to file: ${filename}`);
        fs.writeFileSync(filename, JSON.stringify(materialList, null, 4));
    }
 
    buildDownloadAttribute(imageFormat = 'PNG', imageResolution = '1') {
        return `${imageResolution}K-${imageFormat}`;
    }
 
    splitDownloadAttribute(downloadAttribute) {
        const parts = downloadAttribute.split('-');
    }
 
    getDownloadedMaterialInformation() {
        return {
            filename: this.downloadMaterialFileName,
            content: this.downloadMaterial
        };
    }
 
    clearDownloadMaterial() {
        if (this.downloadMaterial) {
            this.downloadMaterial = null;
        }
        this.downloadMaterialFileName = '';
    }
 
    writeDownloadedMaterialToFile(path = '') {
        const haveDownload = this.downloadMaterialFileName && this.downloadMaterial;
        if (!haveDownload) {
            this.logger.warning('No current material downloaded');
            return;
        }
 
        const filename = `${path}/${this.downloadMaterialFileName}`;
        fs.writeFileSync(filename, this.downloadMaterial);
        this.logger.info(`Saved downloaded material to: ${filename}`);
    }
 
    async downloadMaterialAsset(assetId, imageFormat = 'PNG', imageResolution = '1', downloadAttributeKey = 'downloadAttribute', downloadLinkKey = 'downloadLink') {
        this.clearDownloadMaterial();
 
        const items = this.findMaterial(assetId);
        const target = this.buildDownloadAttribute(imageFormat, imageResolution);
        let url = '';
        let downloadAttribute = '';
 
        items.forEach(item => {
            downloadAttribute = item[downloadAttributeKey];
            if (downloadAttribute === target) {
                url = item[downloadLinkKey];
                this.logger.info(`Found Asset: ${assetId}. Download Attribute: ${downloadAttribute} -> ${url}`);
            }
        });
 
        if (!url) {
            this.logger.error(`No download link found for asset: ${assetId}, attribute: ${target}`);
            return '';
        }
 
        this.downloadMaterialFileName = url.split('file=')[1];
 
        try {
            const response = await fetch(url);
            if (!response.ok) {
                throw new Error(`HTTP error! Status: ${response.status}`);
            }
        
            // Get the response as an ArrayBuffer
            const arrayBuffer = await response.arrayBuffer();
        
            // Convert ArrayBuffer to Buffer
            this.downloadMaterial = Buffer.from(arrayBuffer);
            this.logger.info(`Material file downloaded: ${this.downloadMaterialFileName}`);
 
        } catch (error) {
            this.downloadMaterialFileName = '';
            this.logger.error(`Error occurred while downloading the file: ${error}`);
        }
 
        return this.downloadMaterialFileName;
    }
 
    findMaterial(assetId, key = 'assetId') {
        if (this.materials) {
            return this.materials.filter(item => item[key] === assetId);
        }
        return [];
    }
 
    loadMaterialsList(fileName) {
        this.materials = JSON.parse(fs.readFileSync(fileName, 'utf8'));
        this.logger.info(`Loaded materials list from: ${fileName}`);
        return this.materials;
    }
 
    async downloadMaterialsList() {
        let haveMaterials = false;
        const MATERIALS_CACHE_FILE = 'ambientcg_materials.json';
        // 1. Try to load from cache file
        if (fs.existsSync(MATERIALS_CACHE_FILE)) {
            try {
                const data = fs.readFileSync(MATERIALS_CACHE_FILE, 'utf8');
                this.materials = JSON.parse(data);
                this.logger.info(`Loaded AmbientCG materials from cache: ${MATERIALS_CACHE_FILE}`);
                haveMaterials = true;   
            } catch (e) {
                this.logger.warn(`Failed to load AmbientCG materials cache: ${e.message}`);
            }
        }
 
        // 2. If not in cache, fetch from network
        if (!haveMaterials)
        {
            const headers = { Accept: 'application/csv' };
            const url = new URL('https://ambientCG.com/api/v2/downloads_csv');
            url.searchParams.append('method', 'PBRPhotogrammetry');
            url.searchParams.append('type', 'Material');
            url.searchParams.append('sort', 'Alphabet');
 
            this.logger.info('Downloading materials CSV list from network...');
            try {
                const response = await fetch(url, { headers });
                if (response.status === 200) {
                    const csvContent = await response.text();
                    this.csvMaterials = csvContent;
                    this.materials = parse(csvContent, { columns: true });
                    this.logger.info('Downloaded CSV material list as JSON.');
                } else {
                    this.materials = null;
                    this.logger.warning(`Failed to fetch the CSV material content. HTTP status code: ${response.status}`);
                }
            } catch (error) {
                this.materials = null;
                this.logger.error(`Error downloading materials list: ${error}`);
            }
 
            // Read database list from file.
            this.readDatabaseFromFile('ambientcg_database.json');
            const have_database = this.database && Object.keys(this.database).length > 0;
            if (have_database) {
                for (let material of this.materials) {
                    const assetId = material.assetId;
                    const databaseEntry = this.database.find(entry => entry.assetId === assetId);
                    if (databaseEntry) {
                        // Get "256-PNG" from previewImage (object, not Map)
                        let preview = '';
                        if (databaseEntry.previewImage && typeof databaseEntry.previewImage === 'object') {
                            preview = databaseEntry.previewImage['256-PNG'] || '';
                        }
                        material.displayCategory = databaseEntry.displayCategory;
                        material.previewImage = preview;
                        material.tags = databaseEntry.tags;
                    }
                    else {
                        this.logger.warn('No database entry found for assetId:', assetId);
                    }
            
                }
            }
 
            // Save to cache file after fetching
            try {
                fs.writeFileSync(MATERIALS_CACHE_FILE, JSON.stringify(this.materials, null, 2));
                this.logger.info(`Saved AmbientCG materials to cache: ${MATERIALS_CACHE_FILE}`);
            } catch (e) {
                this.logger.warn(`Failed to write AmbientCG materials cache: ${e.message}`);
            }
        }
 
        //console.log('>>> WRITE MATERIALS TO FILE:', this.materials.length);
        //this.writeMaterialList(this.materials, 'ambientcg_materials.json');
 
        return this.materials;
    }
 
    getDataBase() {
        return this.database;
    }
 
    async downloadAssetDatabase() {
        this.database = {};
 
        haveDatabase = false;
        const MATERIALS_DATABASE_FILE = 'ambientcg_database.json';
        if (fs.existsSync(MATERIALS_DATABASE_FILE)) {
            try {
                const data = fs.readFileSync(MATERIALS_DATABASE_FILE, 'utf8');
                this.database = JSON.parse(data);
                this.logger.info(`Loaded AmbientCG database from file: ${MATERIALS_DATABASE_FILE}`);
                haveDatabase = true;
            } catch (e) {
                this.logger.warn(`Failed to load AmbientCG database from file: ${e.message}`);
            }
        }
 
        if (!haveDatabase) {
 
            const limit = 500
            //https://ambientcg.com/api/v2/full_json?type=material
            const headers = { Accept: 'application/json' };
            let url = new URL('https://ambientcg.com/api/v2/full_json');
            url.searchParams.append('method', 'PBRPhotogrammetry');
            url.searchParams.append('type', 'Material');
            url.searchParams.append('sort', 'Alphabet');        
            url.searchParams.append('limit', limit);
            url.searchParams.append('offset', 0);
            url.searchParams.append('include', 'tagData,previewData')
 
            let numberOfResults = -1;
            let offset = 0
 
            //let data_list = []
            let asset_list = []
            while (numberOfResults === -1 || offset < numberOfResults) {
                this.logger.info(`Downloading asset database from: ${url.toString()}`);
 
                try {
                    const response = await fetch(url, { headers });
                    if (response.status === 200) {
                        const data = await response.json();
                        this.logger.info(`Downloaded data at offset ${offset}. ${data.foundAssets.length} assets found.`);
                        //data_list.push(data);
 
                        // Only keep desired fiels: displayCategory, previewImage, tags and assetId from each entry
                        let reduced_assets = data.foundAssets.map(asset => {
                            return {
                                assetId: asset.assetId,
                                displayCategory: asset.displayCategory,
                                previewImage: asset.previewImage,
                                tags: asset.tags
                            }
                        });
                        asset_list = asset_list.concat(reduced_assets);
 
                        if (numberOfResults === -1) {
                            numberOfResults = data.numberOfResults;
                        }
                        // Update offset for next page
                        if (!data.nextPageHttp) 
                        {
                            break;
                        }
                        offset = offset + limit;
                        
                        url.searchParams.set('offset', offset);
                    } else {
                        this.logger.error(`Status: ${response.status}, ${response.data}`);
                    }
                }
                catch (error) {
                    this.logger.error(`Error downloading asset database: ${error}`);
                }
            }
 
            this.database = asset_list;
 
            // Pull out the preview image link from database and put it
            // into the materials. 
        }
 
        return this.database;
    }
 
    writeDatabaseToFile(filename) {
        if (!this.database) {
            this.logger.info('No database to write');
            return false;
        }
 
        this.logger.info(`Writing database to file: ${filename}`);
        fs.writeFileSync(filename, JSON.stringify(this.database, null, 4));
        return true;
    }
 
    readDatabaseFromFile(filename) {
        try {
            const data = fs.readFileSync(filename, 'utf8');
            this.database = JSON.parse(data);
            return this.database;
        } catch (e) {
            this.logger.error(`Failed to read database from file: ${e.message}`);
            return null;
        }
    }
 
    validateMaterialXDocument(doc) {
        if (!this.mx) {
            this.logger.error('MaterialX module is required');
            return [false, ''];
        }
 
        if (!doc) {
            this.logger.warning('MaterialX document is required');
            return [false, ''];
        }
 
        const valid = doc.validate();
        return [valid, valid ? '' : 'Validation failed'];
    }
 
    addComment(doc, commentString) {
        const comment = doc.addChildOfCategory('comment');
        comment.setDocString(commentString);
    }
 
    getMaterialXString(doc) {
        if (!this.mx) {
            this.logger.error('MaterialX module is required');
            return;
        }
 
        const writeOptions = this.mx.XmlWriteOptions();
        writeOptions.writeXIncludeEnable = false;
        writeOptions.elementPredicate = this.skipLibraryElement;
        return this.mx.writeToXmlString(doc, writeOptions);
    }
}
 
module.exports = new AmbientCGLoader();