Skip to content

refactor: metadata clean up indexes #855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chreman merged 7 commits into from
Feb 6, 2026
Merged

refactor: metadata clean up indexes #855

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3c5f779
refactor: remove indexes
andreishket Jan 12, 2026
22b8479
bugfix: test setup
andreishket Jan 5, 2026
b9df623
Merge remote-tracking branch 'upstream/dev' into dev
chreman Jan 21, 2026
5031590
Merge remote-tracking branch 'upstream/dev' into dev
chreman Jan 28, 2026
383780e
missing configs and parameters for snapshot functionality
chreman Jan 29, 2026
6fe93fc
Merge pull request #854 from OpenKnowledgeMaps/refactor/metadata-clea…
andreishket Feb 3, 2026
677fd03
Merge pull request #853 from OpenKnowledgeMaps/snapshot-bugfixes
chreman Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion server/classes/headstart/preprocessing/Snapshot.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,10 @@ public function __construct($ini_array, $query, $id, $service, $service_name, $v
$snap_php = $ini_array["snapshot"]["snapshot_php"];
$storage = $ini_array["snapshot"]["storage_path"];
$nodemodules = $ini_array["snapshot"]["nodemodules_path"];
$chrome_executable_path = $ini_array["snapshot"]["chrome_executable_path"];

$url = "{$host}{$snap_php}?{$url_postfix}";
$this->cmd = "{$node_path} {$getsvg} \"{$url}\" {$storage}{$post_data['file']}.png {$nodemodules}";
$this->cmd = "{$node_path} {$getsvg} \"{$url}\" {$storage}{$post_data['file']}.png {$nodemodules} {$chrome_executable_path}";
}

public function takeSnapshot() {
Expand Down
2 changes: 2 additions & 0 deletions server/preprocessing/conf/config.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ storage_path = "/path/to/storage/"
snapshot_php = "server/services/snapshot/headstart_snapshot.php"
# snapshot_local_protocol fallback for non-server environments
snapshot_local_protocol = "http://"
# chrome executable path (optional, only needed if puppeteer cannot find chrome automatically)
chrome_executable_path = "/path/to/chrome"


[connection]
Expand Down
1 change: 1 addition & 0 deletions server/preprocessing/other-scripts/base.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ etl <- function(res, repo, non_public) {
subject_cleaned = gsub("(wikidata)?\\.org/entity/[qQ]([\\d]+)?", "", subject_cleaned) # remove wikidata classification
subject_cleaned = gsub("</keyword><keyword>", "", subject_cleaned) # remove </keyword><keyword>
subject_cleaned = gsub("\\[No keyword\\]", "", subject_cleaned)
subject_cleaned = gsub("\\[[^]]*\\]", "", subject_cleaned) # remove any text inside square brackets
subject_cleaned = gsub("\\[[^\\[]+\\][^\\;]+(;|$)?", "", subject_cleaned) # remove classification
subject_cleaned = gsub("[0-9]{2,} [A-Z]+[^;]*(;|$)?", "", subject_cleaned) #remove classification
subject_cleaned = gsub(" -- ", "; ", subject_cleaned) #replace inconsistent keyword separation
Expand Down
6 changes: 5 additions & 1 deletion server/services/getChartSVG.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ function timeout(ms) {

(async() => {
try {
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] });
const browser = await puppeteer.launch({
headless: true,
args: ['--no-sandbox', '--disable-setuid-sandbox'],
executablePath: process.argv[5]
});
await timeout(1000)
const page = await browser.newPage();
await timeout(1000)
Expand Down
3 changes: 3 additions & 0 deletions server/services/search.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,9 @@ function search(
} else {
$vis_type = "overview";
}
if (isset($post_params["vis_type"]) && $post_params["vis_type"] == "geomap") {
$vis_type = "geomap";
}
$snapshot = new \headstart\preprocessing\Snapshot($ini_array, $query, $unique_id, $service, $repo2snapshot[$service], $vis_type);
$snapshot->takeSnapshot();
}
Expand Down
33 changes: 33 additions & 0 deletions server/services/snapshot/data-config_aquanavi.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
var data_config = {
tag: "visualization",
mode: "search_repos",

service: "base",

title: "",
base_unit: "citations",
use_area_uri: true,
show_multiples: false,
show_dropdown: false,
preview_type: "pdf",
sort_options: ["relevance", "title", "authors", "year"],
is_force_areas: true,
language: "eng_pubmed",
area_force_alpha: 0.015,
show_list: true,
content_based: true,
url_prefix: "https://www.base-search.net/Record/",

show_context: true,
create_title_from_context: true,
context_most_relevant_tooltip: true,

doi_outlink: true,
filter_menu_dropdown: true,
sort_menu_dropdown: true,
filter_options: ["all", "open_access"],
//show number of open access documents in context
show_context_oa_number: false,

visualization_type: "geomap", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
};
2 changes: 2 additions & 0 deletions server/services/snapshot/data-config_base.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,6 @@ var data_config = {
filter_options: ["all", "open_access"],
//show number of open access documents in context
show_context_oa_number: false,

visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
};
2 changes: 1 addition & 1 deletion server/services/snapshot/data-config_openaire.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,5 +64,5 @@ var data_config = {
cited_by_tweeters_count: 'tweets',
'readers.mendeley': 'readers'
},

visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
};
1 change: 1 addition & 0 deletions server/services/snapshot/data-config_orcid.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ var data_config = {
filter_options: ["all", "open_access"],
//show number of open access documents in context
show_context_oa_number: false,
visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
};
1 change: 1 addition & 0 deletions server/services/snapshot/data-config_pubmed.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,5 @@ var data_config = {
filter_menu_dropdown: true,
sort_menu_dropdown: true,
filter_options: ["all", "open_access"],
visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
};
8 changes: 6 additions & 2 deletions server/services/snapshot/headstart_snapshot.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
<script type="text/javascript" src="data-config_<?php echo htmlspecialchars($_GET['service']) ?>.js"></script>
<script src="../../../../js/search_options.js"></script>
<script>
vis_type = <?php echo json_encode(isset($_GET['vis_type']) ? $_GET['vis_type'] : "overview") ?>;
data_config.files = [{
title: <?php echo json_encode($_GET['query']) ?>,
file: <?php echo json_encode($_GET['file']) ?>
Expand All @@ -19,13 +20,16 @@
data_config.show_context = true;
data_config.create_title_from_context= true;
data_config.intro = "";
if(<?php echo json_encode(isset($_GET['vis_type']) ? $_GET['vis_type'] : "") ?> === "timeline") {
data_config.is_streamgraph = true;
if (vis_type === "timeline") {
data_config.visualization_type = "timeline";
//data_config.embed_modal = false;
data_config.show_area = false;
} else {
data_config.options = options_<?php echo htmlspecialchars($_GET['service']); ?>.dropdowns;
}
if (vis_type === "geomap") {
data_config.visualization_type = "geomap";
}
</script>

<?php include "../../../dist/headstart.php"; ?>
Expand Down