Skip to content

Commit f60e049

Browse files
jormundur00jormundur00
committed
Add layer building GitHub action
1 parent e64ace9 commit f60e049

File tree

5 files changed

+3632
-0
lines changed

5 files changed

+3632
-0
lines changed

‎.github/workflows/ni-layers.yml

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
#
2+
# Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
3+
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
#
5+
# The Universal Permissive License (UPL), Version 1.0
6+
#
7+
# Subject to the condition set forth below, permission is hereby granted to any
8+
# person obtaining a copy of this software, associated documentation and/or
9+
# data (collectively the "Software"), free of charge and under any and all
10+
# copyright rights in the Software, and any and all patent rights owned or
11+
# freely licensable by each licensor hereunder covering either (i) the
12+
# unmodified Software as contributed to or provided by such licensor, or (ii)
13+
# the Larger Works (as defined below), to deal in both
14+
#
15+
# (a) the Software, and
16+
#
17+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
# one is included with the Software each a "Larger Work" to which the Software
19+
# is contributed by such licensors),
20+
#
21+
# without restriction, including without limitation the rights to copy, create
22+
# derivative works of, display, perform, and distribute the Software and make,
23+
# use, sell, offer for sale, import, export, have made, and have sold the
24+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
# either these or other terms.
26+
#
27+
# This license is subject to the following condition:
28+
#
29+
# The above copyright notice and either this complete permission notice or at a
30+
# minimum a reference to the UPL must be included in all copies or substantial
31+
# portions of the Software.
32+
#
33+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
# SOFTWARE.
40+
#
41+
name: Weekly Native Image Layer Building Tests
42+
43+
on:
44+
push:
45+
paths:
46+
- '.github/workflows/ni-layers.yml'
47+
pull_request:
48+
paths:
49+
- '.github/workflows/ni-layers.yml'
50+
schedule:
51+
- cron: "0 0 * * 1" # Once a week at midnight on Monday
52+
workflow_dispatch:
53+
54+
env:
55+
LIBRARY_METADATA_PATH: ${{ github.workspace }}/vm/tests/gh_workflows/NILayerTests
56+
JAVA_VERSION: 21
57+
PYTHON_VERSION: 3.12.3
58+
59+
jobs:
60+
build-graalvm-and-populate-matrix:
61+
name: Build GraalVM and populate matrix
62+
runs-on: ubuntu-latest
63+
if: (github.repository=='oracle/graal')
64+
outputs:
65+
matrix: ${{ steps.set-matrix.outputs.matrix }}
66+
steps:
67+
- name: Checkout oracle/graal
68+
uses: actions/checkout@v4
69+
- name: Build GraalVM JDK
70+
uses: ./.github/actions/build-graalvm
71+
with:
72+
native-images: 'native-image,native-image-configure,lib:native-image-agent'
73+
components: 'Native Image,Native Image Configure Tool'
74+
java-version: ${{ env.JAVA_VERSION }}
75+
- name: Tar GraalVM JDK
76+
shell: bash
77+
run: tar -czvhf graalvm.tgz -C $(dirname ${GRAALVM_HOME}) $(basename ${GRAALVM_HOME})
78+
- name: Persist GraalVM JDK build
79+
uses: actions/upload-artifact@v4
80+
with:
81+
name: graalvm
82+
path: graalvm.tgz
83+
- name: Setup python
84+
uses: actions/setup-python@v5
85+
with:
86+
python-version: '${{ env.PYTHON_VERSION }}'
87+
- name: Populate matrix
88+
id: set-matrix
89+
run: python3 ${{ env.LIBRARY_METADATA_PATH }}/build_native_image_layer.py ${{ env.LIBRARY_METADATA_PATH }}/
90+
91+
test-native-image-layer-build:
92+
name: ${{ matrix.coordinates }}
93+
runs-on: ubuntu-latest
94+
env:
95+
GRAALVM_HOME: ${{ github.workspace }}/graalvm
96+
timeout-minutes: 20
97+
needs: build-graalvm-and-populate-matrix
98+
strategy:
99+
fail-fast: false
100+
matrix:
101+
coordinates: ${{ fromJson(needs. build-graalvm-and-populate-matrix.outputs.matrix).coordinates }}
102+
steps:
103+
- name: Checkout oracle/graal
104+
uses: actions/checkout@v4
105+
- name: Download GraalVM JDK build
106+
uses: actions/download-artifact@v4
107+
with:
108+
name: graalvm
109+
path: .
110+
- name: Extract GraalVM JDK build
111+
run: tar -xzvf graalvm.tgz -C $(dirname ${GRAALVM_HOME})
112+
- name: "Setup JAVA_HOME"
113+
uses: actions/setup-java@v4
114+
with:
115+
distribution: 'oracle'
116+
java-version: ${{ env.JAVA_VERSION }}
117+
- name: Setup python
118+
uses: actions/setup-python@v5
119+
with:
120+
python-version: '${{ env.PYTHON_VERSION }}'
121+
- name: Build layer
122+
run: |
123+
python3 ${{ env.LIBRARY_METADATA_PATH }}/build_native_image_layer.py ${{ env.GRAALVM_HOME }}/bin/native-image "${{ matrix.coordinates }}"

‎vm/tests/gh_workflows/NILayerTests/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Native Image Layer Tests
2+
3+
The Native Image Layer tests represent a way of automatically testing the Native Image Layer feature with the newest JDKs and the most popular Maven libraries, on a weekly schedule. The tests are run as a weekly GitHub action, defined in [ni-layers.yml](/.github/workflows/ni-layers.yml).
4+
5+
These tests work by running layer-building jobs for each library in [popular-maven-libraries.json](popular-maven-libraries.json), which is automatically generated (Vojin Jovanovic) and updated periodically. The libraries that fail to build are added to [excluded-popular-maven-libraries.json](excluded-popular-maven-libraries.json) and are excluded from future buildings until their issues are fixed.

‎vm/tests/gh_workflows/NILayerTests/build_native_image_layer.py

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#
2+
# Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
3+
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
#
5+
# The Universal Permissive License (UPL), Version 1.0
6+
#
7+
# Subject to the condition set forth below, permission is hereby granted to any
8+
# person obtaining a copy of this software, associated documentation and/or
9+
# data (collectively the "Software"), free of charge and under any and all
10+
# copyright rights in the Software, and any and all patent rights owned or
11+
# freely licensable by each licensor hereunder covering either (i) the
12+
# unmodified Software as contributed to or provided by such licensor, or (ii)
13+
# the Larger Works (as defined below), to deal in both
14+
#
15+
# (a) the Software, and
16+
#
17+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
# one is included with the Software each a "Larger Work" to which the Software
19+
# is contributed by such licensors),
20+
#
21+
# without restriction, including without limitation the rights to copy, create
22+
# derivative works of, display, perform, and distribute the Software and make,
23+
# use, sell, offer for sale, import, export, have made, and have sold the
24+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
# either these or other terms.
26+
#
27+
# This license is subject to the following condition:
28+
#
29+
# The above copyright notice and either this complete permission notice or at a
30+
# minimum a reference to the UPL must be included in all copies or substantial
31+
# portions of the Software.
32+
#
33+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
# SOFTWARE.
40+
#
41+
import json
42+
import os
43+
import subprocess
44+
import sys
45+
from pathlib import Path
46+
47+
def generate_matrix(path_to_data, libs_per_job, delimiter):
48+
'''
49+
Generates a matrix in the format of GAV coordinate tuples (depending on the selected number of libraries per action job) for GitHub actions.
50+
'''
51+
try:
52+
with open(os.path.join(path_to_data, 'popular-maven-libraries.json'), 'r') as f:
53+
data = json.load(f)
54+
with open(os.path.join(path_to_data, 'excluded-popular-maven-libraries.json'), 'r') as f:
55+
exclude_data = json.load(f)
56+
except (FileNotFoundError, json.JSONDecodeError) as e:
57+
print(f"Error loading files: {e}")
58+
sys.exit(1)
59+
60+
matrix = {'coordinates': []}
61+
excluded_coordinates = {f'{lib['group_id']}:{lib['artifact_id']}:{lib['version']}' for lib in exclude_data}
62+
libs_in_job = []
63+
for lib in data:
64+
lib_coordinates = f'{lib['group_id']}:{lib['artifact_id']}:{lib['version']}'
65+
if lib_coordinates in excluded_coordinates:
66+
continue
67+
libs_in_job.append(lib_coordinates)
68+
if len(libs_in_job) == libs_per_job:
69+
matrix['coordinates'].append(delimiter.join(libs_in_job))
70+
libs_in_job = []
71+
72+
if len(libs_in_job) > 0:
73+
matrix['coordinates'].append(delimiter.join(libs_in_job))
74+
75+
try:
76+
github_output = os.getenv('GITHUB_OUTPUT')
77+
if github_output is None:
78+
raise EnvironmentError("GITHUB_OUTPUT environment variable not set")
79+
with open(github_output, 'a') as f:
80+
f.write(f"matrix={json.dumps(matrix)}\n")
81+
except (IOError, EnvironmentError) as e:
82+
print(f"Error writing to GITHUB_OUTPUT: {e}")
83+
sys.exit(1)
84+
85+
def build_layers(native_image_path, coordinates, delimiter):
86+
'''
87+
Builds native-image layers out of the given libraries, given their GAV coordinates and native-image path.
88+
89+
Firstly, the function invokes a maven command to download the library jar with all it's transitive dependencies, given its GAV coordinates.
90+
After that, it invokes a maven command to get the full classpath of the given library.
91+
Finally, it runs a native-image command to build the native-image layer and prints out the used command for local testing in case of issues.
92+
'''
93+
coordinates_list = coordinates.split(delimiter)
94+
95+
for gav in coordinates_list:
96+
currDir = os.getcwd()
97+
group_id, artifact_id, version = gav.rstrip().split(':')
98+
99+
subprocess.run(['mvn', 'dependency:get', f'-Dartifact={gav}', '-Dtransitive=true'])
100+
101+
library_path = os.path.join(Path.home(), '.m2', 'repository', group_id.replace('.','/'), artifact_id, version)
102+
jar_path = os.path.join(library_path, f'{artifact_id}-{version}.jar')
103+
subprocess.run(['cp', f'{os.path.join(library_path, f'{artifact_id}-{version}.pom')}', f'{os.path.join(library_path, 'pom.xml')}'])
104+
105+
if Path(library_path).exists():
106+
subprocess.run(['mkdir', gav])
107+
os.chdir(gav)
108+
image_path = os.getcwd()
109+
os.chdir(library_path)
110+
dependency_path = subprocess.check_output(['mvn', '-q', 'exec:exec', '-Dexec.executable=echo', '-Dexec.args=%classpath']).decode('utf-8').rstrip()
111+
os.chdir(image_path)
112+
command = [native_image_path, '-H:+UnlockExperimentalVMOptions', '-cp' ,f'{jar_path}:{dependency_path}', f'-H:LayerCreate=layer.nil,package={jar_path}', '-H:+ReportExceptionStackTraces', '--no-fallback' , '-o', f'{artifact_id}-{version}'] # Assertions currently excluded, see GR-57236
113+
print(f'Command: {' '.join(command)}')
114+
subprocess.run(command)
115+
os.chdir('..')
116+
117+
os.chdir(currDir)
118+
119+
if __name__ == '__main__':
120+
delimiter = " && "
121+
libs_per_job = 2
122+
if len(sys.argv) == 2:
123+
generate_matrix(sys.argv[1], libs_per_job, delimiter)
124+
elif len(sys.argv) == 3:
125+
build_layers(sys.argv[1], sys.argv[2], delimiter)
126+
else:
127+
print("Error: Wrong number of arguments!")
128+
sys.exit(1)

0 commit comments

Comments
 (0)