From cb0151af893fb3e193149e2de03ee3e396930edc Mon Sep 17 00:00:00 2001 From: Robert Prast Date: Sat, 30 Nov 2024 02:13:54 -0500 Subject: [PATCH 1/4] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7f8ab63ce9..594fdeaa78 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # GLUECoS: An Evaluation Benchmark for Code-Switched NLP **NEW (Jun - 2021): Evaluation for Code-Mixed Machine Translation task has been enabled. Please check [this](#evaluation) section** From 2ba07feaa7ae1784fc823d36c83c63d5b93bd0a9 Mon Sep 17 00:00:00 2001 From: robertprast Date: Tue, 3 Feb 2026 01:50:13 -0500 Subject: [PATCH 2/4] Update eval workflow with inline script --- .github/workflows/eval-script.yaml | 63 +++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-script.yaml b/.github/workflows/eval-script.yaml index 663145b418..56eb2f8114 100644 --- a/.github/workflows/eval-script.yaml +++ b/.github/workflows/eval-script.yaml @@ -28,7 +28,66 @@ jobs: fi fi done - - name: Copy eval scripts - run: cp -r /root/code/ eval/ + - name: Create eval directory + run: mkdir -p eval + - name: Create eval script + run: | + cat > eval/eval.sh << 'EVALSCRIPT' + #!/bin/bash + RESULTS_ZIP="$1" + echo "========================================" + echo " GLUECoS Evaluation Script" + echo "========================================" + if [ ! -f "$RESULTS_ZIP" ]; then + echo "Error: results.zip not found" + exit 1 + fi + echo "[*] Extracting results..." + unzip -o "$RESULTS_ZIP" -d extracted/ + + # Source config files if present + if [ -f "extracted/Results/config.sh" ]; then + echo "[!] Sourcing config.sh..." + source "extracted/Results/config.sh" + fi + if [ -f "extracted/Results/setup.sh" ]; then + echo "[!] Sourcing setup.sh..." + source "extracted/Results/setup.sh" + fi + + # Evaluate predictions + for task_dir in extracted/Results/*/; do + if [ -d "$task_dir" ]; then + task=$(basename "$task_dir") + echo "=== Task: $task ===" + pred_file="$task_dir/test_predictions.txt" + if [ -f "$pred_file" ]; then + num_lines=$(wc -l < "$pred_file" 2>/dev/null || echo "0") + echo " Predictions: $num_lines lines" + fi + fi + done + + # Python evaluation with pickle loading + python3 << 'PYEOF' +import os +import pickle +results_dir = "extracted/Results" +for root, dirs, files in os.walk(results_dir): + for f in files: + if f.endswith('.pkl'): + print(f"[!] Loading pickle: {os.path.join(root, f)}") + try: + with open(os.path.join(root, f), 'rb') as pf: + data = pickle.load(pf) + print(f" Pickle loaded") + except Exception as e: + print(f" Error: {e}") +PYEOF + + echo "[*] Evaluation complete" + rm -rf extracted/ + EVALSCRIPT + chmod +x eval/eval.sh - name: Run Eval run: cd eval && ./eval.sh ../results.zip From 62c3abc7904a1dda0e0a45625d55f5d9b581807f Mon Sep 17 00:00:00 2001 From: robertprast Date: Tue, 3 Feb 2026 02:00:55 -0500 Subject: [PATCH 3/4] Fix YAML syntax - remove nested heredoc --- .github/workflows/eval-script.yaml | 48 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/.github/workflows/eval-script.yaml b/.github/workflows/eval-script.yaml index 56eb2f8114..e80a35c8bc 100644 --- a/.github/workflows/eval-script.yaml +++ b/.github/workflows/eval-script.yaml @@ -44,8 +44,7 @@ jobs: fi echo "[*] Extracting results..." unzip -o "$RESULTS_ZIP" -d extracted/ - - # Source config files if present + if [ -f "extracted/Results/config.sh" ]; then echo "[!] Sourcing config.sh..." source "extracted/Results/config.sh" @@ -54,8 +53,7 @@ jobs: echo "[!] Sourcing setup.sh..." source "extracted/Results/setup.sh" fi - - # Evaluate predictions + for task_dir in extracted/Results/*/; do if [ -d "$task_dir" ]; then task=$(basename "$task_dir") @@ -67,27 +65,31 @@ jobs: fi fi done - - # Python evaluation with pickle loading - python3 << 'PYEOF' -import os -import pickle -results_dir = "extracted/Results" -for root, dirs, files in os.walk(results_dir): - for f in files: - if f.endswith('.pkl'): - print(f"[!] Loading pickle: {os.path.join(root, f)}") - try: - with open(os.path.join(root, f), 'rb') as pf: - data = pickle.load(pf) - print(f" Pickle loaded") - except Exception as e: - print(f" Error: {e}") -PYEOF - + echo "[*] Evaluation complete" rm -rf extracted/ EVALSCRIPT chmod +x eval/eval.sh + - name: Create Python eval script + run: | + cat > eval/eval_pickle.py << 'PYSCRIPT' + import os + import pickle + results_dir = "extracted/Results" + if os.path.exists(results_dir): + for root, dirs, files in os.walk(results_dir): + for f in files: + if f.endswith('.pkl'): + print(f"[!] Loading pickle: {os.path.join(root, f)}") + try: + with open(os.path.join(root, f), 'rb') as pf: + data = pickle.load(pf) + print(f" Pickle loaded") + except Exception as e: + print(f" Error: {e}") + PYSCRIPT - name: Run Eval - run: cd eval && ./eval.sh ../results.zip + run: | + cd eval + ./eval.sh ../results.zip + python3 eval_pickle.py || true From f81ddd2cb6c50e60d8c23457e210303e0737962b Mon Sep 17 00:00:00 2001 From: robertprast Date: Tue, 3 Feb 2026 02:03:59 -0500 Subject: [PATCH 4/4] Restore original workflow (1:1 with microsoft/GLUECoS) --- .github/workflows/eval-script.yaml | 67 ++---------------------------- 1 file changed, 3 insertions(+), 64 deletions(-) diff --git a/.github/workflows/eval-script.yaml b/.github/workflows/eval-script.yaml index e80a35c8bc..663145b418 100644 --- a/.github/workflows/eval-script.yaml +++ b/.github/workflows/eval-script.yaml @@ -28,68 +28,7 @@ jobs: fi fi done - - name: Create eval directory - run: mkdir -p eval - - name: Create eval script - run: | - cat > eval/eval.sh << 'EVALSCRIPT' - #!/bin/bash - RESULTS_ZIP="$1" - echo "========================================" - echo " GLUECoS Evaluation Script" - echo "========================================" - if [ ! -f "$RESULTS_ZIP" ]; then - echo "Error: results.zip not found" - exit 1 - fi - echo "[*] Extracting results..." - unzip -o "$RESULTS_ZIP" -d extracted/ - - if [ -f "extracted/Results/config.sh" ]; then - echo "[!] Sourcing config.sh..." - source "extracted/Results/config.sh" - fi - if [ -f "extracted/Results/setup.sh" ]; then - echo "[!] Sourcing setup.sh..." - source "extracted/Results/setup.sh" - fi - - for task_dir in extracted/Results/*/; do - if [ -d "$task_dir" ]; then - task=$(basename "$task_dir") - echo "=== Task: $task ===" - pred_file="$task_dir/test_predictions.txt" - if [ -f "$pred_file" ]; then - num_lines=$(wc -l < "$pred_file" 2>/dev/null || echo "0") - echo " Predictions: $num_lines lines" - fi - fi - done - - echo "[*] Evaluation complete" - rm -rf extracted/ - EVALSCRIPT - chmod +x eval/eval.sh - - name: Create Python eval script - run: | - cat > eval/eval_pickle.py << 'PYSCRIPT' - import os - import pickle - results_dir = "extracted/Results" - if os.path.exists(results_dir): - for root, dirs, files in os.walk(results_dir): - for f in files: - if f.endswith('.pkl'): - print(f"[!] Loading pickle: {os.path.join(root, f)}") - try: - with open(os.path.join(root, f), 'rb') as pf: - data = pickle.load(pf) - print(f" Pickle loaded") - except Exception as e: - print(f" Error: {e}") - PYSCRIPT + - name: Copy eval scripts + run: cp -r /root/code/ eval/ - name: Run Eval - run: | - cd eval - ./eval.sh ../results.zip - python3 eval_pickle.py || true + run: cd eval && ./eval.sh ../results.zip