yosh/goldberries_data_pipeline
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

frogeline (should be good?)

Browse source
This commit is contained in:
yosh 2024-06-28 05:07:20 -04:00
parent 19bb622074
commit 25ae430488
4 changed files with 5707 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
assign_gb_data.php
View file

@ -5,17 +5,24 @@ $gb_data = json_decode(file_get_contents('gb_data.json'), true);
foreach($arr_camps as &$obj_camp) {
if (!isset($gb_data[$obj_camp->url])) {
$json = json_decode(shell_exec('curl -s "https://api.gamebanana.com/Core/Item/Data?itemtype=Mod&itemid=' . $gb_id . '&fields=name,Owner().name,userid"'));
if (isset($json[0])) {
$gb_data[$obj_camp->url]['name'] = $json[0];
$gb_data[$obj_camp->url]['author_name'] = $json[1];
$gb_data[$obj_camp->url]['author_id'] = $json[2];
if (str_contains($obj_camp->url, '/mods/')) {
$gb_id = preg_replace('/.+\/(\d+)$/', '$1', $obj_camp->url);
$json = json_decode(shell_exec('curl -s "https://api.gamebanana.com/Core/Item/Data?itemtype=Mod&itemid=' . $gb_id . '&fields=name,Owner().name,userid"'));
if (isset($json[0])) {
$gb_data[$obj_camp->url]['name'] = $json[0];
$gb_data[$obj_camp->url]['author_name'] = $json[1];
$gb_data[$obj_camp->url]['author_id'] = $json[2];
} else {
$gb_data[$obj_camp->url]['name'] = 'GB_MOD_REMOVED_NAME';
$gb_data[$obj_camp->url]['author_name'] = 'GB_MOD_REMOVED_AUTHOR_NAME';
$gb_data[$obj_camp->url]['author_id'] = -1;
}
sleep(15);
} else {
$gb_data[$obj_camp->url]['name'] = 'GB_MOD_REMOVED_NAME';
$gb_data[$obj_camp->url]['author_name'] = 'GB_MOD_REMOVED_AUTHOR_NAME';
$gb_data[$obj_camp->url]['author_id'] = -1;
$gb_data[$obj_camp->url]['name'] = 'NOTGB_MOD_NAME';
$gb_data[$obj_camp->url]['author_name'] = 'NOTGB_MOD_AUTHOR_NAME';
$gb_data[$obj_camp->url]['author_id'] = -2;
}
sleep(15);
}
$obj_camp->name = $gb_data[$obj_camp->url]['name'];
$obj_camp->author_id = $gb_data[$obj_camp->url]['author_id'];

5642
gb_data.json Normal file
View file

File diff suppressed because it is too large Load diff

16
process_list.php
View file

@ -64,10 +64,12 @@ foreach ($goldlist[4]['table-row'] as $rowcount => &$rowval) {
// now to iterate over submissions to the map
foreach ($rowval['table-cell'] as $colcount => &$cellval) {
$obj_sub = new Submission();
if (gettype($cellval['p']) !== 'array') {
if (gettype($cellval['p']) === 'string')
echo "LINK MISSING: " . $obj_camp->name . " - " . $obj_map->name . " - " . $cellval['p'] . "\n";
continue;
}
$obj_sub = new Submission();
$obj_sub->proof_url = $cellval['p']['a']['+@xlink:href'];
$tmp_playername = trim($cellval['p']['a']['+content']);
if (preg_match('/ \[FC\]$/', $tmp_playername)) {
@ -179,10 +181,12 @@ foreach ($goldlist[3]['table-row'] as $rowcount => &$rowval) {
$obj_map->challenges[] = $obj_chal;
foreach ($rowval['table-cell'] as $colcount => &$cellval) {
$obj_sub = new Submission();
if (gettype($cellval['p']) !== 'array') {
if (gettype($cellval['p']) === 'string')
echo "LINK MISSING: " . $obj_camp->name . " - " . $obj_map->name . " - " . $cellval['p'] . "\n";
continue;
}
$obj_sub = new Submission();
$obj_sub->proof_url = $cellval['p']['a']['+@xlink:href'];
$tmp_playername = trim($cellval['p']['a']['+content']);
if (preg_match('/ \[FC\]$/', $tmp_playername)) {
@ -233,3 +237,11 @@ complete_collab_parsing($goldlist, $arr_camps, $arr_players, 9, 'Secret Santa Co
// D SIDES PACK
$dside_camp = &find_obj_camp($arr_camps, 'https://gamebanana.com/mods/150759');
complete_collab_parsing($goldlist, $arr_camps, $arr_players, 10, 'D-Sides', 'na', $dside_camp);
file_put_contents('generated_players.json',
json_encode($arr_players, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT)
);
file_put_contents('generated_camps.json',
json_encode($arr_camps, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT)
);

82
process_list_afterjson.php
View file

@ -35,62 +35,40 @@ $id_camp = 1;
$id_map = 1;
$id_chal = 1;
file_put_contents('sql_campaign.sql',
"INSERT INTO Campaign(name, url, author_gb_name, author_gb_id) VALUES\n");
file_put_contents('sql_maps.sql',
"INSERT INTO Map(name, campaign_id) VALUES\n");
file_put_contents('sql_chals.sql',
"INSERT INTO Challenge(map_id, objective_id, difficulty_id, requires_fc, has_fc, is_arbitrary) VALUES\n");
file_put_contents('sql_subs.sql',
"INSERT INTO Submission(challenge_id, player_id, proof_url, is_fc, verifier_notes) VALUES\n");
file_put_contents('sql_player.sql',
"INSERT INTO Player(name) VALUES\n");
$camp_str = "INSERT INTO Campaign(name, url, author_gb_name, author_gb_id) VALUES\n";
$map_str = "INSERT INTO Map(name, campaign_id) VALUES\n";
$chal_str = "INSERT INTO Challenge(map_id, objective_id, difficulty_id, requires_fc, has_fc, is_arbitrary) VALUES\n";
$sub_str = "INSERT INTO Submission(challenge_id, player_id, proof_url, is_fc, verifier_notes) VALUES\n";
$player_str = "INSERT INTO Player(name) VALUES\n";
foreach ($obj_camps as $camp_ind => &$camp) {
file_put_contents('sql_campaign.sql',
"('" . str_replace("'", "''", $camp->name)
$camp_str .= "('" . str_replace("'", "''", $camp->name)
. "', '" . $camp->url
. "', '" . str_replace("'", "''", $camp->author_name)
. "', " . $camp->author_id
. "),\n",
FILE_APPEND);
. "),\n";
// ONLY FOR USE WHEN NOT USING GB VALUES
/*
file_put_contents('sql_campaign.sql',
"('" . str_replace("'", "''", $camp->name)
. "', '" . $camp->url . "', '"
. str_replace("'", "''", "Author Not Retrieved")
. "', " . -1
. "),\n",
FILE_APPEND);
*/
foreach ($camp->maps as $map_ind => &$map) {
file_put_contents('sql_maps.sql',
"('" . str_replace("'", "''", $map->name)
. "', " . $id_camp
. "),\n",
FILE_APPEND);
$map_str .= "('" . str_replace("'", "''", $map->name)
. "', " . $id_camp
. "),\n";
foreach ($map->challenges as $chal_ind => &$chal) {
if (!isset($chal->submissions)) {
continue;
}
file_put_contents('sql_chals.sql',
"(" . $id_map
. ", 1, " . map_diff_id($chal)
. ", " . ($chal->requires_fc ? "'1'" : "'0'")
. ", " . ($chal->has_fc ? "'1'" : "'0'")
. ", '0'),\n",
FILE_APPEND);
$chal_str .= "(" . $id_map
. ", 1, " . map_diff_id($chal)
. ", " . ($chal->requires_fc ? "'1'" : "'0'")
. ", " . ($chal->has_fc ? "'1'" : "'0'")
. ", '0'),\n";
foreach ($chal->submissions as $sub_ind => &$sub) {
file_put_contents('sql_subs.sql',
"(" . $id_chal
. ", " . $sub->player_id
. ", '" . $sub->proof_url
. "', " .($sub->is_fc ? "'1'" : "'0'")
. ", " . (isset($sub->verifier_note) ? ("'" . str_replace("'", "''", $sub->verifier_note) . "'") : "null")
. "),\n",
FILE_APPEND);
$sub_str .= "(" . $id_chal
. ", " . $sub->player_id
. ", '" . $sub->proof_url
. "', " .($sub->is_fc ? "'1'" : "'0'")
. ", " . (isset($sub->verifier_note) ? ("'" . str_replace("'", "''", $sub->verifier_note) . "'") : "null")
. "),\n";
}
$id_chal += 1;
}
@ -100,7 +78,17 @@ foreach ($obj_camps as $camp_ind => &$camp) {
}
foreach ($obj_players as $player_ind => &$player) {
file_put_contents('sql_player.sql',
"('" . str_replace("'", "''", $player) . "'),\n",
FILE_APPEND);
$player_str .= "('" . str_replace("'", "''", $player) . "'),\n";
}
$camp_str = rtrim($camp_str, "\n ,") . ";";
$map_str = rtrim($map_str, "\n ,") . ";";
$chal_str = rtrim($chal_str, "\n ,") . ";";
$sub_str = rtrim($sub_str, "\n ,") . ";";
$player_str = rtrim($player_str, "\n ,") . ";";
file_put_contents('sql_campaign.sql', $camp_str);
file_put_contents('sql_maps.sql', $map_str);
file_put_contents('sql_chals.sql', $chal_str);
file_put_contents('sql_subs.sql', $sub_str);
file_put_contents('sql_player.sql', $player_str);