Merge main, resolve conflicts with vala

1 files changed, 253 insertions, 62 deletions

diff --git a/lib/compilers/dotnet.ts b/lib/compilers/dotnet.ts
index 95ac84ad7..2c9c9a0b4 100644
--- a/lib/compilers/dotnet.ts
+++ b/lib/compilers/dotnet.ts
@@ -40,6 +40,8 @@ import * as exec from '../exec.js';
 import {DotNetAsmParser} from '../parsers/asm-parser-dotnet.js';
 import * as utils from '../utils.js';
 
+const AssemblyName = 'CompilerExplorer';
+
 class DotNetCompiler extends BaseCompiler {
     private readonly sdkBaseDir: string;
     private readonly sdkVersion: string;
@@ -47,10 +49,12 @@ class DotNetCompiler extends BaseCompiler {
     private readonly buildConfig: string;
     private readonly clrBuildDir: string;
     private readonly langVersion: string;
+    private readonly corerunPath: string;
+    private readonly disassemblyLoaderPath: string;
     private readonly crossgen2Path: string;
     private readonly sdkMajorVersion: number;
 
-    private crossgen2VersionString: string;
+    private versionString: string;
 
     constructor(compilerInfo: PreliminaryCompilerInfo, env) {
         super(compilerInfo, env);
@@ -66,9 +70,11 @@ class DotNetCompiler extends BaseCompiler {
         this.clrBuildDir = this.compilerProps<string>(`compiler.${this.compiler.id}.clrDir`);
         this.langVersion = this.compilerProps<string>(`compiler.${this.compiler.id}.langVersion`);
 
+        this.corerunPath = path.join(this.clrBuildDir, 'corerun');
         this.crossgen2Path = path.join(this.clrBuildDir, 'crossgen2', 'crossgen2');
         this.asm = new DotNetAsmParser();
-        this.crossgen2VersionString = '';
+        this.versionString = '';
+        this.disassemblyLoaderPath = path.join(this.clrBuildDir, 'DisassemblyLoader', 'DisassemblyLoader.dll');
     }
 
     get compilerOptions() {
@@ -86,6 +92,9 @@ class DotNetCompiler extends BaseCompiler {
             '--singlemethodgenericarg',
             '--codegenopt',
             '--codegen-options',
+            '--maxgenericcycle',
+            '--maxgenericcyclebreadth',
+            '--max-vectort-bitwidth',
         ];
     }
 
@@ -99,6 +108,11 @@ class DotNetCompiler extends BaseCompiler {
             '--optimize-space',
             '--Ot',
             '--optimize-time',
+            '--enable-generic-cycle-detection',
+            '--inputbubble',
+            '--compilebubblegenerics',
+            '--aot',
+            '--crossgen2',
         ];
     }
 
@@ -107,7 +121,7 @@ class DotNetCompiler extends BaseCompiler {
             <PropertyGroup>
                 <TargetFramework>${this.targetFramework}</TargetFramework>
                 <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-                <AssemblyName>CompilerExplorer</AssemblyName>
+                <AssemblyName>${AssemblyName}</AssemblyName>
                 <LangVersion>${this.langVersion}</LangVersion>
                 <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
                 <Nullable>enable</Nullable>
@@ -115,15 +129,20 @@ class DotNetCompiler extends BaseCompiler {
             </PropertyGroup>
             <ItemGroup>
                 <Compile Include="${sourceFile}" />
+                <Reference Include="DisassemblyLoader" HintPath="${this.disassemblyLoaderPath}" />
             </ItemGroup>
         </Project>
         `;
 
-        const projectFilePath = path.join(programDir, `CompilerExplorer${this.lang.extensions[0]}proj`);
+        const projectFilePath = path.join(programDir, `${AssemblyName}${this.lang.extensions[0]}proj`);
         await fs.writeFile(projectFilePath, projectFileContent);
     }
 
-    setCompilerExecOptions(execOptions: ExecutionOptions, programDir: string) {
+    setCompilerExecOptions(
+        execOptions: ExecutionOptions & {env: Record<string, string>},
+        programDir: string,
+        skipNuget: boolean = false,
+    ) {
         if (!execOptions) {
             execOptions = this.getDefaultExecOptions();
         }
@@ -138,13 +157,16 @@ class DotNetCompiler extends BaseCompiler {
         // directory here.
         execOptions.env.DOTNET_CLI_HOME = programDir;
         execOptions.env.DOTNET_ROOT = path.join(this.clrBuildDir, '.dotnet');
-        // Place nuget packages in the output directory.
-        execOptions.env.NUGET_PACKAGES = path.join(programDir, '.nuget');
         // Try to be less chatty
         execOptions.env.DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 'true';
         execOptions.env.DOTNET_NOLOGO = 'true';
 
         execOptions.customCwd = programDir;
+
+        if (!skipNuget) {
+            // Place nuget packages in the output directory.
+            execOptions.env.NUGET_PACKAGES = path.join(programDir, '.nuget');
+        }
     }
 
     override async buildExecutable(compiler, options, inputFilename, execOptions) {
@@ -152,7 +174,7 @@ class DotNetCompiler extends BaseCompiler {
         const inputFilenameSafe = this.filename(inputFilename);
         const sourceFile = path.basename(inputFilenameSafe);
         await this.writeProjectfile(dirPath, true, sourceFile);
-        return await this.buildToDll(compiler, options, inputFilename, execOptions);
+        return await this.buildToDll(compiler, inputFilename, execOptions);
     }
 
     override async doCompilation(inputFilename, dirPath, key, options, filters, backendOptions, libraries, tools) {
@@ -164,9 +186,8 @@ class DotNetCompiler extends BaseCompiler {
 
     async buildToDll(
         compiler: string,
-        options: string[],
         inputFilename: string,
-        execOptions: ExecutionOptions,
+        execOptions: ExecutionOptions & {env: Record<string, string>},
     ): Promise<CompilationResult> {
         const programDir = path.dirname(inputFilename);
         const nugetConfigPath = path.join(programDir, 'nuget.config');
@@ -194,51 +215,203 @@ class DotNetCompiler extends BaseCompiler {
         return compilerResult;
     }
 
+    async publishAot(
+        compiler: string,
+        ilcOptions: string[],
+        ilcSwitches: string[],
+        inputFilename: string,
+        execOptions: ExecutionOptions & {env: Record<string, string>},
+    ): Promise<CompilationResult> {
+        const programDir = path.dirname(inputFilename);
+
+        this.setCompilerExecOptions(execOptions, programDir, true);
+
+        // serialize ['a', 'b', 'c;d', '*e*'] into a=b;c%3Bd=%2Ae%2A;
+        const msbuildOptions = ilcOptions
+            .map(val => val.replaceAll('*', '%2A').replaceAll(';', '%3B'))
+            .reduce((acc, val, idx) => (idx % 2 === 0 ? (acc ? `${acc}${val}` : `${val}`) : `${acc}=${val};`), '');
+
+        // serialize ['a', 'b', 'c', 'd'] into a;b;c;d
+        const msbuildSwitches = ilcSwitches.join(';');
+
+        const toolVersion = await fs.readFile(`${this.clrBuildDir}/aot/package-version.txt`, 'utf8');
+        const jitOutFile = `${programDir}/jitout`;
+
+        ilcOptions.push('--codegenopt', `JitDisasmAssemblies=${AssemblyName}`);
+
+        await fs.writeFile(
+            path.join(programDir, `${AssemblyName}${this.lang.extensions[0]}proj`),
+            `<Project Sdk="Microsoft.NET.Sdk">
+               <PropertyGroup>
+                 <TargetFramework>${this.targetFramework}</TargetFramework>
+                 <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+                 <AssemblyName>${AssemblyName}</AssemblyName>
+                 <LangVersion>${this.langVersion}</LangVersion>
+                 <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
+                 <Nullable>enable</Nullable>
+                 <OutputType>Library</OutputType>
+                 <RuntimeIdentifier>linux-x64</RuntimeIdentifier>
+               </PropertyGroup>
+               <ItemGroup>
+                 <Compile Include="${path.basename(this.filename(inputFilename))}" />
+                 <PackageReference
+                   Include="Microsoft.DotNet.ILCompiler;runtime.linux-x64.Microsoft.DotNet.ILCompiler"
+                   Version="${toolVersion}" />
+                 <IlcArg Include="${msbuildOptions}${msbuildSwitches};--codegenopt=JitStdOutFile=${jitOutFile}" />
+               </ItemGroup>
+             </Project>`,
+        );
+
+        // prettier-ignore
+        const publishOptions = ['publish', '-c', this.buildConfig, '-v', 'q', '--nologo', '/clp:NoSummary',
+            '--property', 'PublishAot=true', '--packages', `${this.clrBuildDir}/aot`];
+
+        // .NET 7 doesn't support JitStdOutFile, so higher max output is needed for stdout
+        execOptions.maxOutput = 1024 * 1024 * 1024;
+
+        const compilerResult = await super.runCompiler(compiler, publishOptions, inputFilename, execOptions);
+        if (compilerResult.code !== 0) {
+            return compilerResult;
+        }
+
+        await fs.createFile(this.getOutputFilename(programDir, this.outputFilebase));
+
+        const version = '// ilc ' + toolVersion;
+        const output = await fs.readFile(jitOutFile);
+
+        // .NET 7 doesn't support JitStdOutFile, so read from stdout
+        const outputString = output.length ? output.toString().split('\n') : compilerResult.stdout.map(o => o.text);
+
+        await fs.writeFile(
+            this.getOutputFilename(programDir, this.outputFilebase),
+            `${version}\n\n${outputString.reduce((a, n) => `${a}\n${n}`, '')}`,
+        );
+
+        return compilerResult;
+    }
+
     override async runCompiler(
         compiler: string,
         options: string[],
         inputFilename: string,
-        execOptions: ExecutionOptions,
+        execOptions: ExecutionOptions & {env: Record<string, string>},
     ): Promise<CompilationResult> {
-        const crossgen2Options: string[] = [];
-        const configurableOptions = this.configurableOptions;
+        const corerunArgs: string[] = [];
+        // prettier-ignore
+        const toolOptions: string[] = [
+            '--codegenopt',
+            this.sdkMajorVersion === 6 ? 'NgenDisasm=*' : 'JitDisasm=*',
+            '--codegenopt',
+            this.sdkMajorVersion < 8 ? 'JitDiffableDasm=1' : 'JitDisasmDiffable=1',
+            '--parallelism', '1',
+        ];
+        const toolSwitches: string[] = [];
         const programDir = path.dirname(inputFilename);
         const programOutputPath = path.join(programDir, 'bin', this.buildConfig, this.targetFramework);
         const programDllPath = path.join(programOutputPath, 'CompilerExplorer.dll');
+        // prettier-ignore
+        const envVarFileContents = [
+            'DOTNET_EnableWriteXorExecute=0',
+            'DOTNET_JitDisasm=*',
+            'DOTNET_JitDisasmAssemblies=CompilerExplorer',
+            'DOTNET_TieredCompilation=0',
+            this.sdkMajorVersion < 8 ? 'DOTNET_JitDiffableDasm=1' : 'DOTNET_JitDisasmDiffable=1',
+        ];
+        let isAot = false;
+        let isCrossgen2 = this.sdkMajorVersion === 6;
 
-        for (const configurableOption of configurableOptions) {
-            const optionIndex = options.indexOf(configurableOption);
-            if (optionIndex === -1 || optionIndex === options.length - 1) {
+        while (options.length > 0) {
+            const currentOption = options.shift();
+            if (!currentOption) {
                 continue;
             }
-            crossgen2Options.push(options[optionIndex], options[optionIndex + 1]);
-        }
 
-        const configurableSwitches = this.configurableSwitches;
-        for (const configurableSwitch of configurableSwitches) {
-            const switchIndex = options.indexOf(configurableSwitch);
-            if (switchIndex === -1) {
-                continue;
+            if (currentOption === '-e' || currentOption === '--env') {
+                const envVar = options.shift();
+                if (envVar) {
+                    const [name] = envVar.split('=');
+                    const normalizedName = name.trim().toUpperCase();
+                    if (
+                        normalizedName === 'DOTNET_JITDISASM' ||
+                        normalizedName === 'DOTNET_JITDUMP' ||
+                        normalizedName === 'DOTNET_JITDISASMASSEMBILES'
+                    ) {
+                        continue;
+                    }
+                    envVarFileContents.push(envVar);
+                }
+            } else if (currentOption === '-p' || currentOption === '--property') {
+                const property = options.shift();
+                if (property) {
+                    corerunArgs.push('-p', property);
+                }
+            } else if (this.configurableSwitches.indexOf(currentOption) !== -1) {
+                if (currentOption === '--aot') {
+                    isAot = true;
+                } else if (currentOption === '--crossgen2') {
+                    isCrossgen2 = true;
+                } else {
+                    toolSwitches.push(currentOption);
+                }
+            } else if (this.configurableOptions.indexOf(currentOption) !== -1) {
+                const value = options.shift();
+                if (value) {
+                    toolOptions.push(currentOption, value);
+                }
             }
-            crossgen2Options.push(options[switchIndex]);
         }
 
         this.setCompilerExecOptions(execOptions, programDir);
-        const compilerResult = await this.buildToDll(compiler, options, inputFilename, execOptions);
-        if (compilerResult.code !== 0) {
-            return compilerResult;
-        }
+        let compilerResult;
 
-        const crossgen2Result = await this.runCrossgen2(
-            execOptions,
-            this.clrBuildDir,
-            programDllPath,
-            crossgen2Options,
-            this.getOutputFilename(programDir, this.outputFilebase),
-        );
+        if (isAot) {
+            if (!fs.existsSync(`${this.clrBuildDir}/aot`)) {
+                return {
+                    code: -1,
+                    timedOut: false,
+                    didExecute: false,
+                    stdout: [],
+                    stderr: [{text: '--aot is not supported with this version.'}],
+                };
+            }
+
+            compilerResult = await this.publishAot(compiler, toolOptions, toolSwitches, inputFilename, execOptions);
+        } else {
+            compilerResult = await this.buildToDll(compiler, inputFilename, execOptions);
+            if (compilerResult.code !== 0) {
+                return compilerResult;
+            }
 
-        if (crossgen2Result.code !== 0) {
-            return crossgen2Result;
+            if (isCrossgen2) {
+                const crossgen2Result = await this.runCrossgen2(
+                    execOptions,
+                    this.clrBuildDir,
+                    programDllPath,
+                    toolOptions,
+                    toolSwitches,
+                    this.getOutputFilename(programDir, this.outputFilebase),
+                );
+
+                if (crossgen2Result.code !== 0) {
+                    return crossgen2Result;
+                }
+            } else {
+                const envVarFilePath = path.join(programDir, '.env');
+                await fs.writeFile(envVarFilePath, envVarFileContents.join('\n'));
+
+                const corerunResult = await this.runCorerunForDisasm(
+                    execOptions,
+                    this.clrBuildDir,
+                    envVarFilePath,
+                    programDllPath,
+                    corerunArgs,
+                    this.getOutputFilename(programDir, this.outputFilebase),
+                );
+
+                if (corerunResult.code !== 0) {
+                    return corerunResult;
+                }
+            }
         }
 
         return compilerResult;
@@ -256,7 +429,7 @@ class DotNetCompiler extends BaseCompiler {
     ): Promise<BasicExecutionResult> {
         const programDir = path.dirname(executable);
         const programOutputPath = path.join(programDir, 'bin', this.buildConfig, this.targetFramework);
-        const programDllPath = path.join(programOutputPath, 'CompilerExplorer.dll');
+        const programDllPath = path.join(programOutputPath, `${AssemblyName}.dll`);
         const execOptions = this.getDefaultExecOptions();
         execOptions.maxOutput = maxSize;
         execOptions.timeoutMs = this.env.ceProps('binaryExecTimeoutMs', 2000);
@@ -269,9 +442,8 @@ class DotNetCompiler extends BaseCompiler {
         execOptions.env.CORE_ROOT = this.clrBuildDir;
         execOptions.input = executeParameters.stdin;
         const execArgs = ['-p', 'System.Runtime.TieredCompilation=false', programDllPath, ...executeParameters.args];
-        const corerun = path.join(this.clrBuildDir, 'corerun');
         try {
-            const execResult: UnprocessedExecResult = await exec.sandbox(corerun, execArgs, execOptions);
+            const execResult: UnprocessedExecResult = await exec.sandbox(this.corerunPath, execArgs, execOptions);
             return this.processExecutionResult(execResult);
         } catch (err: UnprocessedExecResult | any) {
             if (err.code && err.stderr) {
@@ -287,51 +459,70 @@ class DotNetCompiler extends BaseCompiler {
         }
     }
 
-    async ensureCrossgen2Version(execOptions: ExecutionOptions) {
-        if (!this.crossgen2VersionString) {
-            this.crossgen2VersionString = '// crossgen2 ';
-
+    async checkRuntimeVersion() {
+        if (!this.versionString) {
             const versionFilePath = `${this.clrBuildDir}/version.txt`;
-            const versionResult = await this.exec(this.crossgen2Path, ['--version'], execOptions);
-            if (versionResult.code === 0) {
-                this.crossgen2VersionString += versionResult.stdout;
-            } else if (fs.existsSync(versionFilePath)) {
+            if (fs.existsSync(versionFilePath)) {
                 const versionString = await fs.readFile(versionFilePath);
-                this.crossgen2VersionString += versionString;
+                this.versionString = versionString.toString();
             } else {
-                this.crossgen2VersionString += '<unknown version>';
+                this.versionString = '<unknown version>';
             }
         }
     }
 
+    async runCorerunForDisasm(
+        execOptions: ExecutionOptions,
+        coreRoot: string,
+        envPath: string,
+        dllPath: string,
+        options: string[],
+        outputPath: string,
+    ) {
+        await this.checkRuntimeVersion();
+
+        const corerunOptions = ['--clr-path', coreRoot, '--env', envPath].concat([
+            ...options,
+            this.disassemblyLoaderPath,
+            dllPath,
+        ]);
+        const compilerExecResult = await this.exec(this.corerunPath, corerunOptions, execOptions);
+        const result = this.transformToCompilationResult(compilerExecResult, dllPath);
+
+        await fs.writeFile(
+            outputPath,
+            `// coreclr ${this.versionString}\n\n${result.stdout.map(o => o.text).reduce((a, n) => `${a}\n${n}`, '')}`,
+        );
+
+        return result;
+    }
+
     async runCrossgen2(
         execOptions: ExecutionOptions,
         bclPath: string,
         dllPath: string,
-        options: string[],
+        toolOptions: string[],
+        toolSwitches: string[],
         outputPath: string,
     ) {
-        await this.ensureCrossgen2Version(execOptions);
+        await this.checkRuntimeVersion();
 
-        /* eslint-disable */
+        // prettier-ignore
         const crossgen2Options = [
             '-r', path.join(bclPath, '/'),
+            '-r', this.disassemblyLoaderPath,
             dllPath,
-            '-o', 'CompilerExplorer.r2r.dll',
-            '--codegenopt', this.sdkMajorVersion < 7 ? 'NgenDisasm=*' : 'JitDisasm=*',
-            '--codegenopt', this.sdkMajorVersion < 8 ? 'JitDiffableDasm=1' : 'JitDisasmDiffable=1',
-            '--parallelism', '1',
-            '--inputbubble',
-            '--compilebubblegenerics',
-        ].concat(options);
-        /* eslint-enable */
+            '-o', `${AssemblyName}.r2r.dll`,
+        ].concat(toolOptions).concat(toolSwitches);
 
         const compilerExecResult = await this.exec(this.crossgen2Path, crossgen2Options, execOptions);
         const result = this.transformToCompilationResult(compilerExecResult, dllPath);
 
         await fs.writeFile(
             outputPath,
-            `${this.crossgen2VersionString}\n\n${result.stdout.map(o => o.text).reduce((a, n) => `${a}\n${n}`, '')}`,
+            `// crossgen2 ${this.versionString}\n\n${result.stdout
+                .map(o => o.text)
+                .reduce((a, n) => `${a}\n${n}`, '')}`,
         );
 
         return result;