Skip to content

Commit 453348a

Browse files
DisasterMoDisasterMo
committed
Update the core localisation sync scripts, workflows and instructions
1 parent 9414964 commit 453348a

29 files changed

Lines changed: 2155 additions & 1392 deletions

‎libretro-common/samples/core_options/example_translation/.github/workflows/crowdin_initial_setup.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Uploads source texts and any present translations to Crowdin.
2+
3+
name: Crowdin Translations Initial Setup
4+
5+
# On manual run only.
6+
on:
7+
workflow_dispatch
8+
9+
jobs:
10+
upload_source_file:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- name: Setup Java JDK
14+
uses: actions/setup-java@v4
15+
with:
16+
java-version: 18
17+
distribution: zulu
18+
19+
- name: Setup Python
20+
uses: actions/setup-python@v4
21+
with:
22+
python-version: '3.10'
23+
24+
- name: Checkout
25+
uses: actions/checkout@v3
26+
27+
- name: Initial synchroniation
28+
shell: bash
29+
env:
30+
CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
31+
run: |
32+
python3 intl/initial_sync.py $CROWDIN_API_KEY "<CORE_NAME>" "<PATH/TO>/libretro_core_options.h"

‎libretro-common/samples/core_options/example_translation/.github/workflows/crowdin_source_upload.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Prepare source texts & upload them to Crowdin
2+
3+
name: Crowdin Source Texts Upload
4+
5+
# on change to the English texts
6+
on:
7+
workflow_dispatch
8+
push:
9+
branches:
10+
- master
11+
paths:
12+
- '<PATH/TO>/libretro_core_options.h'
13+
14+
jobs:
15+
upload_source_file:
16+
runs-on: ubuntu-latest
17+
steps:
18+
- name: Setup Java JDK
19+
uses: actions/setup-java@v4
20+
with:
21+
java-version: 18
22+
distribution: zulu
23+
24+
- name: Setup Python
25+
uses: actions/setup-python@v4
26+
with:
27+
python-version: '3.10'
28+
29+
- name: Checkout
30+
uses: actions/checkout@v3
31+
32+
- name: Upload Source
33+
shell: bash
34+
env:
35+
CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
36+
run: |
37+
python3 intl/upload_workflow.py $CROWDIN_API_KEY "<CORE_NAME>" "<PATH/TO>/libretro_core_options.h"

‎libretro-common/samples/core_options/example_translation/.github/workflows/crowdin_translation_sync.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Download translations form Crowdin & Recreate libretro_core_options_intl.h
2+
3+
name: Crowdin Translation Sync
4+
5+
on:
6+
workflow_dispatch
7+
schedule:
8+
# please choose a random time & weekday to avoid all repos synching at the same time
9+
- cron: '<0-59> <0-23> * * 5' # Fridays at , UTC
10+
11+
jobs:
12+
create_intl_file:
13+
permissions:
14+
contents: write # 'write' access to repository contents
15+
pull-requests: write # 'write' access to pull requests
16+
17+
runs-on: ubuntu-latest
18+
steps:
19+
- name: Setup Java JDK
20+
uses: actions/setup-java@v4
21+
with:
22+
java-version: 18
23+
distribution: zulu
24+
25+
- name: Setup Python
26+
uses: actions/setup-python@v4
27+
with:
28+
python-version: '3.10'
29+
30+
- name: Checkout
31+
uses: actions/checkout@v4
32+
with:
33+
persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token.
34+
fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
35+
36+
- name: Create intl file
37+
shell: bash
38+
env:
39+
CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
40+
run: |
41+
python3 intl/download_workflow.py $CROWDIN_API_KEY "<CORE_NAME>" "<PATH/TO>/libretro_core_options_intl.h"
42+
43+
- name: Commit files
44+
run: |
45+
git config --local user.email "[email protected]"
46+
git config --local user.name "github-actions[bot]"
47+
git add intl/*_workflow.py "<PATH/TO>/libretro_core_options_intl.h"
48+
git commit -m "Fetch translations & Recreate libretro_core_options_intl.h"
49+
50+
- name: GitHub Push
51+
uses: ad-m/[email protected]
52+
with:
53+
github_token: ${{ secrets.GITHUB_TOKEN }}
54+
branch: ${{ github.ref }}
55+

‎libretro-common/samples/core_options/example_translation/instructions.txt‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
## Requirements
2+
3+
Make sure the core is libretro conformant:
4+
both `libretro_core_options.h`, containing the English texts, and
5+
`libretro_core_options_intl.h`, containing all already existing
6+
translations, if any, must be present in the same directory.
7+
8+
> Please note: `libretro_core_options_intl.h` does not need to contain
9+
anything, if no translations exist or none of them should be preserved.
10+
11+
The scripts are not compatible with text filled in by macros or during run time.
12+
The procedure should not fail - but those texts will not be made translatable.
13+
14+
Also, please verify the existence and correct use of
15+
16+
`#ifdef HAVE_LANGEXTRA`
17+
18+
and/or
19+
20+
`#ifndef NAVE_NO_LANGEXTRA`
21+
22+
pre-compiler instructions in `libretro_core_options.h` to remove any
23+
references to additional languages on platforms which cannot handle them,
24+
e.g. due to limited RAM.
25+
For an example, refer to an up-to-date core, like [gambatte-libretro](https://github.com/libretro/gambatte-libretro/blob/master/libgambatte/libretro/libretro_core_options.h).
26+
27+
> Make sure `options_intl` in `libretro_core_options.h` correctly references the `intl` options, or the translations will not be applied!
28+
29+
## Adding automatic Crowdin sync
30+
31+
Place the `intl` and `.github` folders, including content, into the root
32+
of the repository.
33+
34+
In `.github/workflows` are two files:
35+
`crowdin_prep.yml` & `crowdin_translate.yml`.
36+
In each of those are placeholders, which need to be replaced.
37+
38+
For convenience, one can run `intl/activate.py`, which will try to find
39+
the `libretro_core_options.h` file as well as identify the core name to
40+
fill those placeholders with.
41+
42+
Even then, one should still check, if it produced the correct result:
43+
44+
For `crowdin_prep.yml`:
45+
> **NOTE:** Please verify, that this workflow watches the correct branch!
46+
Uploads happen, whenever `libretro_core_options.h` of that branch is changed.
47+
48+
- <PATH/TO/libretro_core_options.h FILE> (x2)
49+
- replace with the full path from the root of the repo to the
50+
`libretro_core_options.h` file
51+
52+
- <CORE_NAME>
53+
- the name of the core (or repo)
54+
55+
And for crowdin_translate.yml:
56+
- <0-59> <0-23>
57+
- Minute and hour at which the sync will happen.
58+
The script will generate a random time for this, to avoid
59+
stressing GitHub & Crowdin with many simultaneous runs.
60+
61+
- <CORE_NAME>
62+
- same as above
63+
64+
- <PATH/TO/libretro_core_options_intl.h FILE> (x2)
65+
- replace with the full path from the root of the repo to the
66+
'libretro_core_options_intl.h' file
67+
68+
Create a Pull Request and ask a Crowdin project manager, either on [Crowdin](https://crowdin.com/project/retroarch) or, preferably, on [discord](https://discord.gg/xuMbcVuF) in the `retroarch-translations` channel, to provide you with an access key. Create an Actions repository secret on GitHub named CROWDIN_API_KEY for this access token.
69+
70+
<!-- TODO: set correct permissions https://github.com/marketplace/actions/github-push -->
71+
When everything is ready, run the "Crowdin Translations Initial Setup" workflow manually to upload the source texts and any translations to Crowdin.
72+
73+
> You may either disable the initial workflow or even remove it from your repository. Running it more than once is very much discouraged! That may mess with the newest translations, which are usually not yet incorporated into the repository.
74+
75+
Finally, it is recommended to run the "Crowdin Translation Sync" workflow manually once. If a "Permission to \<repository> denied" error occurs, you might need to configure the GITHUB_TOKEN with the appropriate access rights, [see here](https://github.com/marketplace/actions/github-push#requirements-and-prerequisites).
76+
77+
## (For Crowdin project managers) Creating an access token
78+
79+
To create an access token, navigate to the account settings via your profile picture in the top right. Change to the API tab. Here you should find a `New Token` button.
80+
81+
Name the token after the core/repository, which will receive it. The following permissions should be set:
82+
83+
- Projects
84+
- read
85+
- Source files & strings
86+
- read & write
87+
- Translations
88+
- read & write
89+
- (optional) Translation status
90+
- read
91+
92+
> Please provide these access tokens to the core developers in a private message and delete those after successful setup. Do not share tokens publicly or store them in plain text long term!

‎libretro-common/samples/core_options/example_translation/intl/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
__pycache__
2+
crowdin-cli.jar
3+
*.h
4+
*.json

‎libretro-common/samples/core_options/example_translation/intl/activate.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/usr/bin/env python3
2+
3+
import os
4+
import glob
5+
import random as r
6+
7+
# -------------------- MAIN -------------------- #
8+
9+
if __name__ == '__main__':
10+
DIR_PATH = os.path.dirname(os.path.realpath(__file__))
11+
if os.path.basename(DIR_PATH) != "intl":
12+
raise RuntimeError("Script is not in intl folder!")
13+
14+
BASE_PATH = os.path.dirname(DIR_PATH)
15+
WORKFLOW_PATH = os.path.join(BASE_PATH, ".github", "workflows")
16+
PREP_WF = os.path.join(WORKFLOW_PATH, "crowdin_prep.yml")
17+
TRANSLATE_WF = os.path.join(WORKFLOW_PATH, "crowdin_translate.yml")
18+
CORE_NAME = os.path.basename(BASE_PATH)
19+
CORE_OP_FILE = os.path.join(BASE_PATH, "**", "libretro_core_options.h")
20+
21+
core_options_hits = glob.glob(CORE_OP_FILE, recursive=True)
22+
23+
if len(core_options_hits) == 0:
24+
raise RuntimeError("libretro_core_options.h not found!")
25+
elif len(core_options_hits) > 1:
26+
print("More than one libretro_core_options.h file found:\n\n")
27+
for i, file in enumerate(core_options_hits):
28+
print(f"{i} {file}\n")
29+
30+
while True:
31+
user_choice = input("Please choose one ('q' will exit): ")
32+
if user_choice == 'q':
33+
exit(0)
34+
elif user_choice.isdigit():
35+
core_op_file = core_options_hits[int(user_choice)]
36+
break
37+
else:
38+
print("Please make a valid choice!\n\n")
39+
else:
40+
core_op_file = core_options_hits[0]
41+
42+
core_intl_file = os.path.join(os.path.dirname(core_op_file.replace(BASE_PATH, ''))[1:],
43+
'libretro_core_options_intl.h')
44+
core_op_file = os.path.join(os.path.dirname(core_op_file.replace(BASE_PATH, ''))[1:],
45+
'libretro_core_options.h')
46+
minutes = r.randrange(0, 59, 5)
47+
hour = r.randrange(0, 23)
48+
49+
with open(PREP_WF, 'r') as wf_file:
50+
prep_txt = wf_file.read()
51+
52+
prep_txt = prep_txt.replace("<CORE_NAME>", CORE_NAME)
53+
prep_txt = prep_txt.replace("<PATH/TO>/libretro_core_options.h",
54+
core_op_file)
55+
with open(PREP_WF, 'w') as wf_file:
56+
wf_file.write(prep_txt)
57+
58+
59+
with open(TRANSLATE_WF, 'r') as wf_file:
60+
translate_txt = wf_file.read()
61+
62+
translate_txt = translate_txt.replace('<0-59>', f"{minutes}")
63+
translate_txt = translate_txt.replace('<0-23>', f"{hour}")
64+
translate_txt = translate_txt.replace('# Fridays at , UTC',
65+
f"# Fridays at {hour%12}:{minutes if minutes > 9 else '0' + str(minutes)} {'AM' if hour < 12 else 'PM'}, UTC")
66+
translate_txt = translate_txt.replace("<CORE_NAME>", CORE_NAME)
67+
translate_txt = translate_txt.replace('<PATH/TO>/libretro_core_options_intl.h',
68+
core_intl_file)
69+
with open(TRANSLATE_WF, 'w') as wf_file:
70+
wf_file.write(translate_txt)

libretro-common/samples/core_options/example_translation/translation scripts/intl/core_option_regex.py renamed to libretro-common/samples/core_options/example_translation/intl/core_option_regex.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
import re
22

3-
# 0: full struct; 1: up to & including first []; 2: content between first {}
4-
p_struct = re.compile(r'(struct\s*[a-zA-Z0-9_\s]+\[])\s*'
5-
r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+)\s*)*'
3+
# 0: full struct; 1: up to & including first []; 2 & 3: comments; 4: content between first {}
4+
p_struct = re.compile(r'(\bstruct\b\s*[a-zA-Z0-9_\s]+\[])\s*' # 1st capturing group
5+
r'(?:(?=(\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+))\2\s*)*' # 2nd capturing group
66
r'=\s*' # =
7-
r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+)\s*)*'
7+
r'(?:(?=(\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+))\3\s*)*' # 3rd capturing group
88
r'{((?:.|[\r\n])*?)\{\s*NULL,\s*NULL,\s*NULL\s*(?:.|[\r\n])*?},?(?:.|[\r\n])*?};') # captures full struct, it's beginning and it's content
99
# 0: type name[]; 1: type; 2: name
10-
p_type_name = re.compile(r'(retro_core_option_[a-zA-Z0-9_]+)\s*'
11-
r'(option_cats([a-z_]{0,8})|option_defs([a-z_]{0,8}))\s*\[]')
10+
p_type_name = re.compile(r'(\bretro_core_option_[a-zA-Z0-9_]+)\s*'
11+
r'(\boption_cats([a-z_]{0,8})|\boption_defs([a-z_]*))\s*\[]')
1212
# 0: full option; 1: key; 2: description; 3: additional info; 4: key/value pairs
1313
p_option = re.compile(r'{\s*' # opening braces
1414
r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
@@ -76,9 +76,9 @@
7676

7777
p_masked = re.compile(r'([A-Z_][A-Z0-9_]+)\s*(\"(?:"\s*"|\\\s*|.)*\")')
7878

79-
p_intl = re.compile(r'(struct retro_core_option_definition \*option_defs_intl\[RETRO_LANGUAGE_LAST]) = {'
79+
p_intl = re.compile(r'(\bstruct retro_core_option_definition \*option_defs_intl\[RETRO_LANGUAGE_LAST]) = {'
8080
r'((?:.|[\r\n])*?)};')
81-
p_set = re.compile(r'static INLINE void libretro_set_core_options\(retro_environment_t environ_cb\)'
81+
p_set = re.compile(r'\bstatic INLINE void libretro_set_core_options\(retro_environment_t environ_cb\)'
8282
r'(?:.|[\r\n])*?};?\s*#ifdef __cplusplus\s*}\s*#endif')
8383

8484
p_yaml = re.compile(r'"project_id": "[0-9]+".*\s*'

0 commit comments

Comments
 (0)