fix: support PEP 735 arrays without raising runtime requirements

packages/pyright-internal/package.json

@@ -15,7 +15,7 @@
         "test:coverage": "jest --forceExit --reporters=jest-junit --reporters=default --coverage --coverageReporters=cobertura --coverageReporters=html --coverageReporters=json"
     },
     "dependencies": {
-        "@iarna/toml": "2.2.5",
+        "@iarna/toml": "3.0.0",
         "@yarnpkg/fslib": "2.10.1",
         "@yarnpkg/libzip": "2.2.4",
         "chalk": "^4.1.2",

packages/pyright-internal/src/analyzer/service.ts

@@ -1072,7 +1072,8 @@ export class AnalyzerService {
     private _parsePyprojectTomlFile(pyprojectPath: string): object | undefined {
         return this._attemptParseFile(pyprojectPath, (fileContents, attemptCount) => {
             try {
-                const configObj = TOML.parse(fileContents);
+                // NOTE(scip-python): @iarna/toml 3 rejects CRLF in comments. Do not strip lone CR.
+                const configObj = TOML.parse(fileContents.replace(/\r\n/g, '\n'));
                 if (configObj && configObj.tool && (configObj.tool as TOML.JsonMap).pyright) {
                     return (configObj.tool as TOML.JsonMap).pyright as object;
                 }

packages/pyright-internal/src/tests/config.test.ts

@@ -8,6 +8,8 @@
  */
 
 import assert from 'assert';
+import * as fs from 'fs';
+import * as os from 'os';
 
 import { AnalyzerService } from '../analyzer/service';
 import { CommandLineOptions } from '../common/commandLineOptions';
@@ -280,16 +282,26 @@ test('AutoSearchPathsOnAndExtraPaths', () => {
 });
 
 test('BasicPyprojectTomlParsing', () => {
-    const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_with_pyproject_toml'));
-    const service = new AnalyzerService('<default>', createFromRealFileSystem(), options);
-    const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ true);
-
-    service.setOptions(commandLineOptions);
-
-    const configOptions = service.test_getConfigOptions(commandLineOptions);
-    assert.strictEqual(configOptions.defaultPythonVersion!, PythonVersion.V3_9);
-    assert.strictEqual(configOptions.diagnosticRuleSet.reportMissingImports, 'error');
-    assert.strictEqual(configOptions.diagnosticRuleSet.reportUnusedClass, 'warning');
+    // NOTE(scip-python): Exercise mixed arrays and comments with both LF and CRLF.
+    const fixture = fs.readFileSync('src/tests/samples/project_with_pyproject_toml/pyproject.toml', 'utf8');
+    for (const newline of ['\n', '\r\n']) {
+        const cwd = normalizePath(fs.mkdtempSync(combinePaths(os.tmpdir(), 'pyright-toml-')));
+        const service = new AnalyzerService('<default>', createFromRealFileSystem(), options);
+        try {
+            const content =
+                '# configuration\n' + fixture.replace('pythonVersion = "3.9"', 'pythonVersion = "3.9" # inline');
+            fs.writeFileSync(combinePaths(cwd, 'pyproject.toml'), content.replace(/\r?\n/g, newline));
+            const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ true);
+            service.setOptions(commandLineOptions);
+            const configOptions = service.test_getConfigOptions(commandLineOptions);
+            assert.strictEqual(configOptions.defaultPythonVersion!, PythonVersion.V3_9);
+            assert.strictEqual(configOptions.diagnosticRuleSet.reportMissingImports, 'error');
+            assert.strictEqual(configOptions.diagnosticRuleSet.reportUnusedClass, 'warning');
+        } finally {
+            service.dispose();
+            fs.rmSync(cwd, { recursive: true, force: true });
+        }
+    }
 });
 
 test('FindFilesInMemoryOnly', () => {

packages/pyright-internal/src/tests/samples/project_with_pyproject_toml/pyproject.toml

@@ -5,3 +5,9 @@ reportMissingImports = true
 reportUnusedClass = "warning"
 
 pythonVersion = "3.9"
+
+[dependency-groups]
+dev = [{include-group = "test"}, "requests"]
+
+[tool.example]
+max_id = 9223372036854775807

packages/pyright-scip/package.json

@@ -46,7 +46,7 @@
         "webpack-cli": "^4.9.1"
     },
     "dependencies": {
-        "@iarna/toml": "^2.2.5",
+        "@iarna/toml": "^3.0.0",
         "command-exists": "^1.2.9",
         "commander": "^9.2.0",
         "diff": "^5.0.0",

packages/pyright-scip/snapshots/input/pyproject_1/lib/helpers.py

@@ -0,0 +1,2 @@
+def value():
+    return 42

packages/pyright-scip/snapshots/input/pyproject_1/main.py

@@ -1,4 +1,6 @@
+from helpers import value
+
 def main():
-    pass
+    return value()
 
 main()

packages/pyright-scip/snapshots/input/pyproject_1/pyproject.toml

@@ -3,3 +3,16 @@ name = "abc"
 
 [tool.poetry]
 version = "16.05"
+
+[dependency-groups]
+dev = [{include-group = "test"}, "requests"]
+
+[tool.example]
+max_id = 9223372036854775807
+
+[tool.pyright]
+exclude = ["main.py"]
+
+[tool.scip]
+include = ["main.py"]
+extraPaths = ["lib"]

packages/pyright-scip/snapshots/output/pyproject_1/lib/helpers.py

@@ -0,0 +1,6 @@
+# < definition scip-python python abc 16.05 helpers/__init__:
+
+def value():
+#   ^^^^^ definition  abc 16.05 helpers/value().
+    return 42
+

packages/pyright-scip/snapshots/output/pyproject_1/main.py

@@ -1,8 +1,13 @@
 # < definition scip-python python abc 16.05 main/__init__:
 
+from helpers import value
+#    ^^^^^^^ reference  abc 16.05 helpers/__init__:
+#                   ^^^^^ reference  abc 16.05 helpers/value().
+
 def main():
 #   ^^^^ definition  abc 16.05 main/main().
-    pass
+    return value()
+#          ^^^^^ reference  abc 16.05 helpers/value().
 
 main()
 #^^^ reference  abc 16.05 main/main().

packages/pyright-scip/src/config.ts

@@ -434,7 +434,8 @@ export class ScipPyrightConfig {
         return this._attemptParseFile(pyprojectPath, (fileContents, attemptCount) => {
             try {
                 // First, try and load tool.scip section
-                const configObj = TOML.parse(fileContents);
+                // @iarna/toml 3 rejects CRLF inside comments; preserve lone CR for validation.
+                const configObj = TOML.parse(fileContents.replace(/\r\n/g, '\n'));
                 if (configObj && configObj.tool && (configObj.tool as TOML.JsonMap).scip) {
                     return (configObj.tool as TOML.JsonMap).scip as object;
                 }

packages/pyright-scip/src/indexer.ts

@@ -46,7 +46,8 @@ export class Indexer {
         try {
             const pyprojectTomlContents = getPyprojectTomlContents();
             if (pyprojectTomlContents) {
-                const tomlMap = TOML.parse(pyprojectTomlContents);
+                // @iarna/toml 3 rejects CRLF inside comments; normalize valid line endings only.
+                const tomlMap = TOML.parse(pyprojectTomlContents.replace(/\r\n/g, '\n'));
                 // See: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#specification
                 let project = tomlMap['project'] as TOML.JsonMap | undefined;
                 if (project) {

packages/pyright-scip/test/test-main.ts

@@ -274,6 +274,48 @@ version = "16.05"`,
     }
 }
 
+function testPyprojectLineEndings(): void {
+    const fixture = path.resolve('snapshots/input/pyproject_1');
+    for (const newline of ['\n', '\r\n']) {
+        // macOS temporary directories can be symlinked; match the analyzer's resolved file paths.
+        const projectRoot = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'scip-toml-')));
+        try {
+            fs.mkdirSync(path.join(projectRoot, 'lib'));
+            for (const file of ['main.py', 'lib/helpers.py']) {
+                fs.copyFileSync(path.join(fixture, file), path.join(projectRoot, file));
+            }
+            const content =
+                '# metadata\n' +
+                fs
+                    .readFileSync(path.join(fixture, 'pyproject.toml'), 'utf8')
+                    .replace('name = "abc"', 'name = "abc" # inline');
+            fs.writeFileSync(path.join(projectRoot, 'pyproject.toml'), content.replace(/\r?\n/g, newline));
+            indexAction({
+                projectName: '',
+                projectVersion: '',
+                cwd: projectRoot,
+                output: 'index.scip',
+                environment: path.resolve('snapshots/testEnv.json'),
+                infer: { projectVersionFromCommit: false },
+                quiet: true,
+                dev: false,
+                showProgressRateLimit: undefined,
+            });
+            const index = scip.Index.deserializeBinary(fs.readFileSync(path.join(projectRoot, 'index.scip')));
+            const main = index.documents.find((document) => document.relative_path === 'main.py');
+            assert.ok(main, 'tool.scip must take precedence over the tool.pyright exclusion');
+            assert.strictEqual(
+                main!.occurrences.filter(
+                    (occurrence) => occurrence.symbol === 'scip-python python abc 16.05 helpers/value().'
+                ).length,
+                2
+            );
+        } finally {
+            fs.rmSync(projectRoot, { recursive: true, force: true });
+        }
+    }
+}
+
 function testSourceSearchPaths(): void {
     for (const testCase of [
         { config: undefined, expected: 'src_handler' },
@@ -381,6 +423,7 @@ function testMissingProjectVersion(): void {
 
 function unitTests(): void {
     testPyprojectParsing();
+    testPyprojectLineEndings();
     testSourceSearchPaths();
     testMissingProjectVersion();
     pythonCLITests();