Merge pull request #367 from JarvusInnovations/feat/docker-lensing

feat: docker-based lensing

Author: themightychris

lib/Lens.js

@@ -44,12 +44,10 @@ class Lens extends Configurable {
         const config = await super.getConfig();
 
         // process lens configuration
-        if (!config.package) {
-            throw new Error(`hololens has no package defined: ${this.name}`);
+        if (config.package) {
+            config.command = config.command || 'lens-tree {{ input }}';
         }
 
-        config.command = config.command || 'lens-tree {{ input }}';
-
         if (config.before) {
             config.before =
                 typeof config.before == 'string'
@@ -103,7 +101,62 @@ class Lens extends Configurable {
     async buildSpec (inputTree) {
         const config = await this.getCachedConfig();
 
+        if (config.container) {
+            return this.buildSpecForContainer(inputTree, config);
+        } else if (config.package) {
+            return this.buildSpecForHabitatPackage(inputTree, config);
+        } else {
+            throw new Error(`hololens has no package or container defined: ${this.name}`);
+        }
+    }
+
+    async buildSpecForContainer (inputTree, config) {
+        const { container: containerQuery } = config;
+
+        // check if image exists locally first
+        let imageHash;
+        try {
+            const inspectOutput = await Studio.execDocker(['inspect', containerQuery]);
+            const imageInfo = JSON.parse(inspectOutput)[0];
+            imageHash = imageInfo.Id;
+            logger.info(`found local image: ${containerQuery}@${imageHash}`);
+        } catch (err) {
+            // image doesn't exist locally or can't be inspected, try pulling
+            logger.info(`pulling image: ${containerQuery}`);
+
+            try {
+                await Studio.execDocker(['pull', containerQuery], { $relayStdout: true });
+                const inspectOutput = await Studio.execDocker(['inspect', containerQuery]);
+                const imageInfo = JSON.parse(inspectOutput)[0];
+                imageHash = imageInfo.Id;
+            } catch (err) {
+                throw new Error(`failed to pull container image ${containerQuery}: ${err.message}`);
+            }
+        }
+
+        if (!imageHash) {
+            throw new Error(`failed to get hash for container image ${containerQuery}`);
+        }
+
+        // build spec
+        const data = {
+            ...config,
+            container: `${containerQuery.replace(/:.*$/, '')}@${imageHash}`,
+            input: await inputTree.write(),
+            output: null,
+            before: null,
+            after: null
+        };
 
+        // write spec and return packet
+        return {
+            ...await SpecObject.write(this.workspace.getRepo(), 'lens', data),
+            data,
+            type: 'container'
+        };
+    }
+
+    async buildSpecForHabitatPackage (inputTree, config) {
         // determine current package version
         const { package: packageQuery } = config;
         const [pkgOrigin, pkgName, pkgVersion, pkgBuild] = packageQuery.split('/');
@@ -163,14 +216,6 @@ class Lens extends Configurable {
         }
 
 
-        // old studio method that might be useful as fallback/debug option
-        // const setupOutput = await studio.exec('hab', 'pkg', 'install', 'core/hab-plan-build');
-        // const originOutput = await studio.exec('hab', 'origin', 'key', 'generate', 'holo');
-        // const buildOutput = await studio.habPkgExec('core/hab-plan-build', 'hab-plan-build', '/src/lenses/compass');
-        // const studio = await Studio.get(this.workspace.getRepo().gitDir);
-        // let packageIdent = await studio.getPackage(packageQuery);
-
-
         // build spec
         const data = {
             ...config,
@@ -185,21 +230,21 @@ class Lens extends Configurable {
         // write spec and return packet
         return {
             ...await SpecObject.write(this.workspace.getRepo(), 'lens', data),
-            data
+            data,
+            type: 'habitat'
         };
     }
 
-    async executeSpec (specHash, options) {
-        return Lens.executeSpec(specHash, {...options, repo: this.workspace.getRepo()});
+    async executeSpec (specType, specHash, options) {
+        return Lens.executeSpec(specType, specHash, {...options, repo: this.workspace.getRepo()});
     }
 
-    static async executeSpec (specHash, { refresh=false, save=true, repo=null, cacheFrom=null, cacheTo=null }) {
+    static async executeSpec (specType, specHash, options) {
+        const { refresh=false, cacheFrom=null, cacheTo=null, save=true } = options;
 
-        // load holorepo
-        if (!repo) {
-            repo = await Repo.getFromEnvironment();
-        }
 
+        // load holorepo
+        const repo = options.repo || await Repo.getFromEnvironment();
         const git = await repo.getGit();
 
 
@@ -228,9 +273,161 @@ class Lens extends Configurable {
         }
 
 
-        // ensure the rest runs inside a studio environment
+        // execute lens in container or with habitat package:
+        let lensedTreeHash;
+        if (specType == 'container') {
+            lensedTreeHash = await Lens.executeSpecForContainer(repo, specHash);
+        } else if (specType == 'habitat') {
+            lensedTreeHash = await Lens.executeSpecForHabitatPackage(repo, specHash);
+        }
+
+        // save ref to accelerate next projection
+        if (save) {
+            await git.updateRef(specRef, lensedTreeHash);
+
+            if (cacheTo) {
+                await _cacheResultTo(repo, specRef, cacheTo);
+            }
+        }
+
+        return lensedTreeHash;
+    }
+
+    static async executeSpecForContainer (repo, specHash) {
+        const git = await repo.getGit();
+
+        // read and parse spec file
+        const specToml = await git.catFile({ p: true }, specHash);
+        const {
+            holospec: {
+                lens: spec
+            }
+        } = TOML.parse(specToml);
+
+        // write commit with input tree and spec content
+        const commitHash = await git.commitTree(spec.input, {
+            p: [],
+            m: specToml
+        });
+
+        // extract repository and hash from container string
+        const containerMatch = spec.container.match(/^.+@sha256:([a-f0-9]{64})$/);
+        if (!containerMatch) {
+            throw new Error(`Invalid container format: ${spec.container}`);
+        }
+        const [, sha256Hash] = containerMatch;
+
+        // create and start container
+        const persistentDebugContainer = process.env.HOLO_DEBUG_PERSIST_CONTAINER;
+        let containerId;
+        try {
+            if (persistentDebugContainer) {
+                try {
+                    const containerInfo = await Studio.execDocker(['inspect', persistentDebugContainer]);
+                    const containerState = JSON.parse(containerInfo)[0].State;
+
+                    if (containerState.Running) {
+                        logger.info(`Found running debug container: ${persistentDebugContainer}`);
+                        containerId = persistentDebugContainer;
+                    }
+                } catch (error) {
+                    containerId = null;
+                }
+            }
+
+            // create container
+            if (!containerId) {
+                containerId = await Studio.execDocker([
+                    'create',
+                    '-p', '9000:9000',
+                    ...(persistentDebugContainer ? ['--name', persistentDebugContainer] : []),
+                    ...(process.env.DEBUG ? ['-e', 'DEBUG=1'] : []),
+                    sha256Hash
+                ]);
+                containerId = containerId.trim();
+
+                logger.info('starting container');
+                await Studio.execDocker(['start', containerId]);
+            }
+
+            // wait for port 9000 to be available
+            let attempts = 0;
+            const maxAttempts = 30;
+            const waitTime = 1000; // 1 second
+
+            while (attempts < maxAttempts) {
+                try {
+                    const containerInfo = await Studio.execDocker(['inspect', containerId]);
+                    const containerState = JSON.parse(containerInfo)[0].State;
+
+                    if (containerState.Running) {
+                        // check if port 9000 is listening
+                        try {
+                            await Studio.execDocker([
+                                'exec',
+                                containerId,
+                                'nc',
+                                '-z',
+                                'localhost',
+                                '9000'
+                            ]);
+                            break;
+                        } catch (err) {
+                            // ignore error and continue waiting
+                        }
+                    }
+                } catch (err) {
+                    // ignore error and continue waiting
+                }
+
+                await new Promise(resolve => setTimeout(resolve, waitTime));
+                attempts++;
+            }
+
+            if (attempts >= maxAttempts) {
+                throw new Error('Timeout waiting for git server to be ready');
+            }
+
+            // push commit to git server
+            logger.info(`pushing and executing job: ${commitHash}`);
+            await git.push(`http://localhost:9000/`, `${commitHash}:refs/heads/lens-input`, {
+                force: true,
+                $wait: true,
+                $onStderr: (line) => process.stderr.write(`\x1b[90m${line}\x1b[0m\n`)
+            });
+
+            // fetch and verify output commit
+            const outputRef = `refs/lens-jobs/${specHash}`;
+            logger.info('fetching result');
+            await git.fetch('http://localhost:9000/', `+refs/heads/lens-input:${outputRef}`);
+
+            // verify the output commit's parent matches our input commit
+            const outputParent = await git.revParse(`${outputRef}^`);
+            if (outputParent !== commitHash) {
+                throw new Error(`Output commit parent ${outputParent} does not match input commit ${commitHash}`);
+            }
+
+            return await git.getTreeHash(outputRef);
+
+        } finally {
+            // cleanup
+            if (containerId && !persistentDebugContainer) {
+                try {
+                    await Studio.execDocker(['stop', containerId]);
+                    await Studio.execDocker(['rm', containerId]);
+                } catch (err) {
+                    logger.warn(`Failed to cleanup container: ${err.message}`);
+                }
+            }
+        }
+    }
+
+    static async executeSpecForHabitatPackage (repo, specHash) {
+        const git = await repo.getGit();
+
         let lensedTreeHash;
 
+        // ensure the rest runs inside a studio environment
         if (!await Studio.isEnvironmentStudio()) {
             const studio = await Studio.get(repo.gitDir);
             lensedTreeHash = await studio.holoLensExec(specHash);
@@ -307,16 +504,6 @@ class Lens extends Configurable {
         }
 
 
-        // save ref to accelerate next projection
-        if (save) {
-            await git.updateRef(specRef, lensedTreeHash);
-
-            if (cacheTo) {
-                await _cacheResultTo(repo, specRef, cacheTo);
-            }
-        }
-
-
         // return tree hash
         return lensedTreeHash;
     }

lib/Projection.js

@@ -162,11 +162,11 @@ class Projection {
 
             // build tree of matching files to input to lens
             logger.info(`building input tree for lens ${lens.name} from ${inputRoot == '.' ? '' : (path.join(inputRoot, '.')+'/')}{${inputFiles}}`);
-            const { hash: specHash } = await lens.buildSpec(await lens.buildInputTree(this.output.root));
+            const { hash: specHash, type: specType } = await lens.buildSpec(await lens.buildInputTree(this.output.root));
 
 
             // check for existing output tree
-            const outputTreeHash = await lens.executeSpec(specHash, { cacheFrom, cacheTo });
+            const outputTreeHash = await lens.executeSpec(specType, specHash, { cacheFrom, cacheTo });
 
 
             // verify output