generate-copyright: Now generates a library file too.

We only run reuse once, so the output has to be filtered to find only the files that are relevant to the library tree.

Outputs build/COPYRIGHT.html and build/COPYRIGHT-library.html.

Author: jonathanpallant

src/bootstrap/src/core/build_steps/run.rs

@@ -211,12 +211,13 @@ impl Step for GenerateCopyright {
     fn run(self, builder: &Builder<'_>) -> Self::Output {
         let license_metadata = builder.ensure(CollectLicenseMetadata);
 
-        // Temporary location, it will be moved to the proper one once it's accurate.
         let dest = builder.out.join("COPYRIGHT.html");
+        let dest_libstd = builder.out.join("COPYRIGHT-library.html");
 
         let mut cmd = builder.tool_cmd(Tool::GenerateCopyright);
         cmd.env("LICENSE_METADATA", &license_metadata);
         cmd.env("DEST", &dest);
+        cmd.env("DEST_LIBSTD", &dest_libstd);
         cmd.env("OUT_DIR", &builder.out);
         cmd.env("CARGO", &builder.initial_cargo);
         cmd.run(builder);

src/tools/generate-copyright/src/cargo_metadata.rs

@@ -52,14 +52,13 @@ pub struct PackageMetadata {
 /// assume `reuse` has covered it already.
 pub fn get_metadata_and_notices(
     cargo: &Path,
-    dest: &Path,
+    vendor_path: &Path,
     root_path: &Path,
     manifest_paths: &[&Path],
 ) -> Result<BTreeMap<Package, PackageMetadata>, Error> {
     let mut output = get_metadata(cargo, root_path, manifest_paths)?;
 
     // Now do a cargo-vendor and grab everything
-    let vendor_path = dest.join("vendor");
     println!("Vendoring deps into {}...", vendor_path.display());
     run_cargo_vendor(cargo, &vendor_path, manifest_paths)?;
 

src/tools/generate-copyright/src/main.rs

@@ -6,63 +6,96 @@ use rinja::Template;
 
 mod cargo_metadata;
 
-#[derive(Template)]
-#[template(path = "COPYRIGHT.html")]
-struct CopyrightTemplate {
-    in_tree: Node,
-    dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
-}
-
 /// The entry point to the binary.
 ///
 /// You should probably let `bootstrap` execute this program instead of running it directly.
 ///
 /// Run `x.py run generate-copyright`
 fn main() -> Result<(), Error> {
     let dest_file = env_path("DEST")?;
+    let libstd_dest_file = env_path("DEST_LIBSTD")?;
     let out_dir = env_path("OUT_DIR")?;
     let cargo = env_path("CARGO")?;
     let license_metadata = env_path("LICENSE_METADATA")?;
 
-    let collected_tree_metadata: Metadata =
-        serde_json::from_slice(&std::fs::read(&license_metadata)?)?;
-
     let root_path = std::path::absolute(".")?;
-    let workspace_paths = [
-        Path::new("./Cargo.toml"),
-        Path::new("./src/tools/cargo/Cargo.toml"),
-        Path::new("./library/Cargo.toml"),
-    ];
-    let mut collected_cargo_metadata =
-        cargo_metadata::get_metadata_and_notices(&cargo, &out_dir, &root_path, &workspace_paths)?;
 
-    let stdlib_set =
-        cargo_metadata::get_metadata(&cargo, &root_path, &[Path::new("./library/std/Cargo.toml")])?;
+    // Scan Cargo dependencies
+    let mut collected_cargo_metadata =
+        cargo_metadata::get_metadata_and_notices(&cargo, &out_dir.join("vendor"), &root_path, &[
+            Path::new("./Cargo.toml"),
+            Path::new("./src/tools/cargo/Cargo.toml"),
+            Path::new("./library/Cargo.toml"),
+        ])?;
+
+    let library_collected_cargo_metadata = cargo_metadata::get_metadata_and_notices(
+        &cargo,
+        &out_dir.join("library-vendor"),
+        &root_path,
+        &[Path::new("./library/Cargo.toml")],
+    )?;
 
     for (key, value) in collected_cargo_metadata.iter_mut() {
-        value.is_in_libstd = Some(stdlib_set.contains_key(key));
+        value.is_in_libstd = Some(library_collected_cargo_metadata.contains_key(key));
     }
 
+    // Load JSON output by reuse
+    let collected_tree_metadata: Metadata =
+        serde_json::from_slice(&std::fs::read(&license_metadata)?)?;
+
+    // Find libstd sub-set
+    let library_collected_tree_metadata = Metadata {
+        files: collected_tree_metadata
+            .files
+            .trim_clone(&Path::new("./library"), &Path::new("."))
+            .unwrap(),
+    };
+
+    // Output main file
     let template = CopyrightTemplate {
         in_tree: collected_tree_metadata.files,
         dependencies: collected_cargo_metadata,
     };
-
     let output = template.render()?;
-
     std::fs::write(&dest_file, output)?;
 
+    // Output libstd subset file
+    let template = LibraryCopyrightTemplate {
+        in_tree: library_collected_tree_metadata.files,
+        dependencies: library_collected_cargo_metadata,
+    };
+    let output = template.render()?;
+    std::fs::write(&libstd_dest_file, output)?;
+
     Ok(())
 }
 
+/// The HTML template for the toolchain copyright file
+#[derive(Template)]
+#[template(path = "COPYRIGHT.html")]
+struct CopyrightTemplate {
+    in_tree: Node,
+    dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
+}
+
+/// The HTML template for the library copyright file
+#[derive(Template)]
+#[template(path = "COPYRIGHT-library.html")]
+struct LibraryCopyrightTemplate {
+    in_tree: Node,
+    dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
+}
+
 /// Describes a tree of metadata for our filesystem tree
-#[derive(serde::Deserialize)]
+///
+/// Must match the JSON emitted by the `CollectLicenseMetadata` bootstrap tool.
+#[derive(serde::Deserialize, Clone, Debug, PartialEq, Eq)]
 struct Metadata {
     files: Node,
 }
 
 /// Describes one node in our metadata tree
-#[derive(serde::Deserialize, rinja::Template)]
+#[derive(serde::Deserialize, rinja::Template, Clone, Debug, PartialEq, Eq)]
 #[serde(rename_all = "kebab-case", tag = "type")]
 #[template(path = "Node.html")]
 pub(crate) enum Node {
@@ -72,8 +105,74 @@ pub(crate) enum Node {
     Group { files: Vec<String>, directories: Vec<String>, license: License },
 }
 
+impl Node {
+    /// Clone, this node, but only if the path to the item is within the match path
+    fn trim_clone(&self, match_path: &Path, parent_path: &Path) -> Option<Node> {
+        match self {
+            Node::Root { children } => {
+                let mut filtered_children = Vec::new();
+                for node in children {
+                    if let Some(child_node) = node.trim_clone(match_path, parent_path) {
+                        filtered_children.push(child_node);
+                    }
+                }
+                if filtered_children.is_empty() {
+                    None
+                } else {
+                    Some(Node::Root { children: filtered_children })
+                }
+            }
+            Node::Directory { name, children, license } => {
+                let child_name = parent_path.join(name);
+                if !(child_name.starts_with(match_path) || match_path.starts_with(&child_name)) {
+                    return None;
+                }
+                let mut filtered_children = Vec::new();
+                for node in children {
+                    if let Some(child_node) = node.trim_clone(match_path, &child_name) {
+                        filtered_children.push(child_node);
+                    }
+                }
+                Some(Node::Directory {
+                    name: name.clone(),
+                    children: filtered_children,
+                    license: license.clone(),
+                })
+            }
+            Node::File { name, license } => {
+                let child_name = parent_path.join(name);
+                if !(child_name.starts_with(match_path) || match_path.starts_with(&child_name)) {
+                    return None;
+                }
+                Some(Node::File { name: name.clone(), license: license.clone() })
+            }
+            Node::Group { files, directories, license } => {
+                let mut filtered_child_files = Vec::new();
+                for child in files {
+                    let child_name = parent_path.join(child);
+                    if child_name.starts_with(match_path) || match_path.starts_with(&child_name) {
+                        filtered_child_files.push(child.clone());
+                    }
+                }
+                let mut filtered_child_dirs = Vec::new();
+                for child in directories {
+                    let child_name = parent_path.join(child);
+                    if child_name.starts_with(match_path) || match_path.starts_with(&child_name) {
+                        filtered_child_dirs.push(child.clone());
+                    }
+                }
+                Some(Node::Group {
+                    files: filtered_child_files,
+                    directories: filtered_child_dirs,
+                    license: license.clone(),
+                })
+            }
+        }
+    }
+}
+
 /// A License has an SPDX license name and a list of copyright holders.
-#[derive(serde::Deserialize)]
+#[derive(serde::Deserialize, Clone, Debug, PartialEq, Eq)]
 struct License {
     spdx: String,
     copyright: Vec<String>,

src/tools/generate-copyright/templates/COPYRIGHT-library.html

@@ -0,0 +1,53 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Copyright notices for The Rust Standard Library</title>
+</head>
+<body>
+
+<h1>Copyright notices for The Rust Standard Library</h1>
+
+<p>This file describes the copyright and licensing information for the Rust
+Standard Library source code within The Rust Project git tree, and the
+third-party dependencies used when building the Rust Standard Library.</p>
+
+<h2>Table of Contents</h2>
+<ul>
+    <li><a href="#in-tree-files">In-tree files</a></li>
+    <li><a href="#out-of-tree-dependencies">Out-of-tree dependencies</a></li>
+</ul>
+
+<h2 id="in-tree-files">In-tree files</h2>
+
+<p>The following licenses cover the in-tree source files that were used in this
+release:</p>
+
+{{ in_tree|safe }}
+
+<h2 id="out-of-tree-dependencies">Out-of-tree dependencies</h2>
+
+<p>The following licenses cover the out-of-tree crates that were used in the
+Rust Standard Library in this release:</p>
+
+{% for (key, value) in dependencies %}
+    <h3>📦 {{key.name}}-{{key.version}}</h3>
+    <p><b>URL:</b> <a href="https://crates.io/crates/{{ key.name }}/{{ key.version }}">https://crates.io/crates/{{ key.name }}/{{ key.version }}</a></p>
+    <p><b>Authors:</b> {{ value.authors|join(", ") }}</p>
+    <p><b>License:</b> {{ value.license }}</p>
+    {% let len = value.notices.len() %}
+    {% if len > 0 %}
+        <p><b>Notices:</b>
+        {% for (notice_name, notice_text) in value.notices %}
+            <details>
+                <summary><code>{{ notice_name }}</code></summary>
+                <pre>
+{{ notice_text }}
+                </pre>
+            </details>
+        {% endfor %}
+        </p>
+    {% endif %}
+{% endfor %}
+</body>
+</html>