Batch analyze licenses + compare changes. (#1184)

* Batch analyze licenses + compare changes.

* Update third_party/pub/README.md

Co-authored-by: Jonas Finnemann Jensen <jopsen@gmail.com>

* Update third_party/pub/download_pub_licenses.sh

Co-authored-by: Jonas Finnemann Jensen <jopsen@gmail.com>

* Updated based on review comments.

* Explicit output file name.

* Update .gitignore

Co-authored-by: Jonas Finnemann Jensen <jopsen@gmail.com>

Co-authored-by: Jonas Finnemann Jensen <jopsen@gmail.com>

Author: isoos

tool/license_detection/README.md

@@ -0,0 +1,11 @@
+# Evaluate license detection changes
+
+The tools in this directory help to evaluate changes in the license-detection code,
+by downloading licenses from `pub.dev` and running a difference on the license
+detection output before and after the changes.
+
+The cached licenses are stored in `.dart_tool/pana/license-cache/`.
+
+ - Use `download_pub_dev_licenses.sh` to populate the directory with fresh licenses from `pub.dev`.
+ - Run `batch_analyse_licenses.dart` before and after a license detection change.
+ - Run `compare_analysis.dart` with the before and after file of the prior change.

tool/license_detection/batch_analyse_licenses.dart

@@ -0,0 +1,44 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:pana/src/license.dart';
+
+Future<void> main(List<String> args) async {
+  if (args.isEmpty || args.contains('--help')) {
+    print('dart batch_analyse_licenses.dart <output.json>');
+    print('');
+    print(
+        'Read all license files from the license cache directory and create an '
+        'aggregated summary of the results.');
+    return;
+  }
+  final outputFileName = args.single;
+
+  final files = Directory('.dart_tool/pana/license-cache')
+      .listSync()
+      .whereType<File>()
+      .toList();
+  files.sort((a, b) => a.path.compareTo(b.path));
+  final result = <String, dynamic>{};
+  for (final file in files) {
+    try {
+      final content = file.readAsStringSync();
+      final list =
+          await detectLicenseInContent(content, relativePath: 'LICENSE');
+      final spdxIds = list.map((e) => e.spdxIdentifier).toList()..sort();
+      final packageName =
+          file.path.split('/').last.split('LICENSE-').last.split('.txt').first;
+      result[packageName] = {
+        'spdxIds': spdxIds,
+      };
+    } catch (_) {
+      // TODO: also track errors
+    }
+  }
+  await File(outputFileName)
+      .writeAsString(const JsonEncoder.withIndent('  ').convert(result));
+}

tool/license_detection/compare_analysis.dart

@@ -0,0 +1,59 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+Future<void> main(List<String> args) async {
+  if (args.isEmpty || args.contains('--help')) {
+    print('dart batch_compare_analysis.dart <before.json> <after.json>');
+    print('');
+    print(
+        'Compare analysis results and highlight changes using the output of the '
+        '`batch_compare_licenses.dart script.');
+    return;
+  }
+  final a = await _read(args[0]);
+  final b = await _read(args[1]);
+
+  final changes = <String, List<String>>{};
+  final keys = a.keys.toSet().intersection(b.keys.toSet());
+  for (final key in keys) {
+    final diff = a[key]!.diff(b[key]!);
+    for (final d in diff) {
+      changes.putIfAbsent(d, () => []).add(key);
+    }
+  }
+  final entries = changes.entries.toList()
+    ..sort((a, b) => -a.value.length.compareTo(b.value.length));
+  for (final entry in entries) {
+    print(
+        '${entry.value.length.toString().padLeft(6)} ${entry.key.padLeft(30)}: ${entry.value.take(5).join(', ')}');
+  }
+}
+
+class LicenseResult {
+  final List<String> spdxIds;
+
+  LicenseResult({required this.spdxIds});
+  factory LicenseResult.fromJson(Map<String, dynamic> input) {
+    return LicenseResult(
+        spdxIds:
+            (input['spdxIds'] as List?)?.cast<String>() ?? const <String>[]);
+  }
+
+  List<String> diff(LicenseResult other) {
+    return <String>[
+      ...spdxIds.where((id) => !other.spdxIds.contains(id)).map((e) => '-$e'),
+      ...other.spdxIds.where((id) => !spdxIds.contains(id)).map((e) => '+$e'),
+    ];
+  }
+}
+
+Future<Map<String, LicenseResult>> _read(String inputFilePath) async {
+  final content = await File(inputFilePath).readAsString();
+  final data = json.decode(content) as Map<String, dynamic>;
+  return data.map((key, value) =>
+      MapEntry(key, LicenseResult.fromJson(value as Map<String, dynamic>)));
+}

tool/license_detection/download_pub_dev_licenses.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PROJECT_DIR="$( cd ${SCRIPT_DIR}/../.. && pwd )"
+LICENSES_DIR="${PROJECT_DIR}/.dart_tool/pana/license-cache"
+
+mkdir -p "${LICENSES_DIR}"
+cd "${LICENSES_DIR}"
+
+# Get all package names
+get_all_package_names() { curl -s  -H 'Accept-Encoding: gzip' 'https://pub.dev/api/package-names' | gzip -d | jq -r .packages[]; }
+
+# Given a package name, get archive URL for latest version
+get_archive_url() { curl -sL "https://pub.dev/api/packages/$1" | jq -r .latest.archive_url; }
+
+# Given a package name, get LICENSE file from latest version
+get_license() { curl -sL $(get_archive_url "$1") | tar -xzO --ignore-case LICENSE 2> /dev/null; }
+
+# Given a package name, download license to LICENSE-<package>.txt
+download_license() { get_license "$1" > "LICENSE-$1.txt"; }
+
+export -f get_all_package_names
+export -f get_archive_url
+export -f get_license
+export -f download_license
+
+get_all_package_names | parallel -j 20 download_license