Skip to content

Update coreprofiler db #7341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Update coreprofiler db #7341

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5a85338
update coreprofiler DM with version 1.1.6
hugolefeuvre Oct 6, 2025
7a0a3f6
Merge branch 'galaxyproject:main' into update-coreprofiler-db
hugolefeuvre Oct 6, 2025
21dba93
remove tokens in params and add bash variables instead
hugolefeuvre Oct 10, 2025
b918916
add real bash variables and error/help message if bash variables are …
hugolefeuvre Oct 27, 2025
b2c8622
change test value for bigsDB
hugolefeuvre Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 59 additions & 59 deletions ...data_manager_build_coreprofiler/data_manager/data_manager_build_coreprofiler_download.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,29 +30,27 @@
#set $scheme_name = str($coreprofiler_scheme_select).split('-')[0]
#set $scheme_db = str($coreprofiler_scheme_select).split('-')[3]

#set $db_version = str($version.db_version)

mkdir -p '$out_file.extra_files_path' &&
mkdir -p coreprofiler_${scheme_name}/scheme_$scheme_name/ &&
mkdir -p coreprofiler_${scheme_name}_${db_version}/scheme_$scheme_name/ &&

#if str($scheme_db) == "enterobase":
coreprofiler db download
-s $scheme_name
-o coreprofiler_${scheme_name}/scheme_$scheme_name
#else:
coreprofiler db download
-s $scheme_name
-o coreprofiler_${scheme_name}/scheme_$scheme_name
-k $consumer_token
-ks $consumer_secret
-a $access_token
-as $access_secret
-o coreprofiler_${scheme_name}_${db_version}/scheme_$scheme_name
#if str($version.db_version) == "token_version"
-k ${CONSUMER_TOKEN}
-ks ${CONSUMER_SECRET}
-a ${ACCESS_TOKEN}
-as ${ACCESS_SECRET}
#end if
&&
coreprofiler db makeblastdb
-s coreprofiler_${scheme_name}/scheme_$scheme_name
-s coreprofiler_${scheme_name}_${db_version}/scheme_$scheme_name
-n $scheme_name
-p coreprofiler_${scheme_name}/db_$scheme_name &&
-p coreprofiler_${scheme_name}_${db_version}/db_$scheme_name &&

mv coreprofiler_${scheme_name} '$out_file.extra_files_path' &&
mv coreprofiler_${scheme_name}_${db_version} '$out_file.extra_files_path' &&

cp '$dmjson' '$out_file'
]]></command>
Expand All @@ -65,21 +63,31 @@ cp '$dmjson' '$out_file'
#set $scheme_loci = str($coreprofiler_scheme_select).split('-')[2]
#set $scheme_db = str($coreprofiler_scheme_select).split('-')[3]

#set $db_version = str($version.db_version)

{
"data_tables":{
"coreprofiler_scheme":[
{
"value": "coreprofiler_downloaded_#echo date.today().strftime('%d%m%Y')#-${scheme_name}-${scheme_desc}-${scheme_loci}-${scheme_db}",
"name": "${scheme_name}: ${scheme_desc} [${scheme_loci} loci] (${scheme_db})",
"path": "coreprofiler_${scheme_name}",
"database": "coreprofiler_${scheme_name}/db_${scheme_name}/${scheme_name}.fasta",
"scheme": "coreprofiler_${scheme_name}/scheme_${scheme_name}"
"value": "coreprofiler_downloaded_#echo date.today().strftime('%d%m%Y')#-${scheme_name}-${scheme_desc}-${scheme_loci}-${scheme_db}-${db_version}",
"name": "${scheme_name}: ${scheme_desc} [${scheme_loci} loci] (${scheme_db}-${db_version})",
"path": "coreprofiler_${scheme_name}_${db_version}",
"database": "coreprofiler_${scheme_name}_${db_version}/db_${scheme_name}/${scheme_name}.fasta",
"scheme": "coreprofiler_${scheme_name}_${db_version}/scheme_${scheme_name}"
}
]
}
}]]></configfile>
</configfiles>
<inputs>
<conditional name="version">
<param name="db_version" type="select" label="Select if you want latest version from pubMLST or BigsDB using tokens" help="If you choose latest, you will have to provide tokens and secrets to access the database following the procedure describe in the documentation." >
<option value="token" selected="true">Latest version from pubMLST or BigsDB, you will need to set bash variables for tokens and secrets as described in the documentation.</option>
<option value="no_token">No need of tokens but it will download previous version from pubMLST or BigsDB (last update January 2025) or if you want a database from Enterobase</option>
</param>
<when value="token"/>
<when value="no_token"/>
</conditional>
<param name="coreprofiler_scheme_select" type="select" label="CoreProfiler available scheme" help="Choose a schema from a reference platform supported in CoreProfiler">
<option value="bordetella_1-cgMLST_genus-1415-BIGSdb">bordetella_1: cgMLST_genus [1415 loci] (BIGSdb)</option>
<option value="bordetella_4-cgMLST_pertussis-2038-BIGSdb">bordetella_4: cgMLST_pertussis [2038 loci] (BIGSdb)</option>
Expand Down Expand Up @@ -133,61 +141,40 @@ cp '$dmjson' '$out_file'
<option value="vparahaemolyticus_3-cgMLST-2254-pubmlst">vparahaemolyticus_3: cgMLST [2254 loci] (pubmlst)</option>
<option value="xcitri_1-cgMLST-1618-pubmlst">xcitri_1: cgMLST [1618 loci] (pubmlst)</option>
</param>
<param name="consumer_token" type="text" label="Consumer token" help="Your consumer token to access the database (only for pubMLST and BigsDB scheme)." optional="true"/>
<param name="consumer_secret" type="text" label="Consumer secret" help="Your consumer secret to access the database (only for pubMLST and BigsDB scheme)." optional="true"/>
<param name="access_token" type="text" label="Access token" help="Your access token to download a specific scheme (only for pubMLST and BigsDB scheme)." optional="true"/>
<param name="access_secret" type="text" label="Access secret" help="Your access secret to download a specific scheme (only for pubMLST and BigsDB scheme)." optional="true"/>
</inputs>
<outputs>
<data name="out_file" format="data_manager_json" label="${tool.name}"/>
</outputs>
<tests>
<test expect_num_outputs="1">
<param name="coreprofiler_scheme_select" value="borrelia_3-cgMLST-639-pubmlst" />
<!-- The next 4 parameteres may need to be changed with your own tokens and secrets or with most recents tokens and secrets -->
<param name="consumer_token" value="0pwHPAPemYbD5PXE4zchczMa" />
<param name="consumer_secret" value="ToMyhH5jCCY1OADzid5dbD3l4oVEoSeUNy5Q5QkI7j" />
<param name="access_token" value="XWvU5fa2B6aEAAQJMUdSdilybnp4mOZc" />
<param name="access_secret" value="Cce0OGiVdoj3uGclDenSw4TEQXIKHt2k" />
<conditional name="version">
<param name="db_version" value="no_token"/>
</conditional>
<output name="out_file">
<assert_contents>
<has_text text='"coreprofiler_scheme":'/>
<has_text_matching expression='"value": "coreprofiler_downloaded_[0-9]{8}-borrelia_3-cgMLST-639-pubmlst"'/>
<has_text text='"name": "borrelia_3: cgMLST [639 loci] (pubmlst)'/>
<has_text text='"path": "coreprofiler_borrelia_3"'/>
<has_text text='"database": "coreprofiler_borrelia_3/db_borrelia_3/borrelia_3.fasta"'/>
<has_text text='"scheme": "coreprofiler_borrelia_3/scheme_borrelia_3"'/>
<has_text_matching expression='"value": "coreprofiler_downloaded_[0-9]{8}-borrelia_3-cgMLST-639-pubmlst-no_token"'/>
<has_text text='"name": "borrelia_3: cgMLST [639 loci] (pubmlst-no_token)"'/>
<has_text text='"path": "coreprofiler_borrelia_3_no_token"'/>
<has_text text='"database": "coreprofiler_borrelia_3_no_token/db_borrelia_3/borrelia_3.fasta"'/>
<has_text text='"scheme": "coreprofiler_borrelia_3_no_token/scheme_borrelia_3"'/>
</assert_contents>
</output>
</test>
<test expect_num_outputs="1">
<param name="coreprofiler_scheme_select" value="bordetella_1-cgMLST_genus-1415-BIGSdb" />
<!-- The next 4 parameteres may need to be changed with your own tokens and secrets or with most recents tokens and secrets -->
<param name="consumer_token" value="N83HAcCf337oxFebWJcEZxt6" />
<param name="consumer_secret" value="678IqYFaeSD6D2zvTShyTiSkAylVTTt1o3F7P63aMZ" />
<param name="access_token" value="P15sZxOP0o0tlEAsfYvmJOMVe2sS7T4N" />
<param name="access_secret" value="XIu2ZkDhMSCbiAtFR8ELIc5mt35cWVox" />
<output name="out_file">
<assert_contents>
<has_text text='"coreprofiler_scheme":'/>
<has_text_matching expression='"value": "coreprofiler_downloaded_[0-9]{8}-bordetella_1-cgMLST_genus-1415-BIGSdb"'/>
<has_text text='"name": "bordetella_1: cgMLST_genus [1415 loci] (BIGSdb)"'/>
<has_text text='"path": "coreprofiler_bordetella_1"'/>
<has_text text='"database": "coreprofiler_bordetella_1/db_bordetella_1/bordetella_1.fasta"'/>
<has_text text='"scheme": "coreprofiler_bordetella_1/scheme_bordetella_1"'/>
</assert_contents>
</output>
</test>
<test expect_num_outputs="1">
<param name="coreprofiler_scheme_select" value="escherichia_v1-cgMLST-2513-enterobase" />
<conditional name="version">
<param name="db_version" value="no_token"/>
</conditional>
<output name="out_file">
<assert_contents>
<has_text text='"coreprofiler_scheme":'/>
<has_text_matching expression='"value": "coreprofiler_downloaded_[0-9]{8}-escherichia_v1-cgMLST-2513-enterobase"'/>
<has_text text='"name": "escherichia_v1: cgMLST [2513 loci] (enterobase)'/>
<has_text text='"path": "coreprofiler_escherichia_v1"'/>
<has_text text='"database": "coreprofiler_escherichia_v1/db_escherichia_v1/escherichia_v1.fasta"'/>
<has_text text='"scheme": "coreprofiler_escherichia_v1/scheme_escherichia_v1"'/>
<has_text_matching expression='"value": "coreprofiler_downloaded_[0-9]{8}-bordetella_1-cgMLST_genus-1415-BIGSdb-no_token"'/>
<has_text text='"name": "bordetella_1: cgMLST_genus [1415 loci] (BIGSdb-no_token)"'/>
<has_text text='"path": "coreprofiler_bordetella_1_no_token"'/>
<has_text text='"database": "coreprofiler_bordetella_1_no_token/db_bordetella_1/bordetella_1.fasta"'/>
<has_text text='"scheme": "coreprofiler_bordetella_1_no_token/scheme_bordetella_1"'/>
</assert_contents>
</output>
</test>
Expand All @@ -201,7 +188,7 @@ Please refer to this page for details on how to use the tool and which schema op

Use Galaxy's data manager framework to download and install new CoreProfiler schemes :

If you want to use a scheme from enterobase, you do not need to provide any token or secret, just leave the fields empty.
If you want to use a scheme from enterobase, you do not need to provide any token or secret.

However, if you want to use a scheme from **pubMLST** or **BigsDB**, you will need to follow a procedure before launching the data manager.

Expand Down Expand Up @@ -233,7 +220,14 @@ After authorizing, it will give you a verification code that you need to enter i

It will then return your access token and secret.

5. Provide the consumer token, consumer secret, access token, and access secret in the data manager tool.
5. Provide the consumer token, consumer secret, access token, and access secret in the data manager tool by setting this bash variables :

"""
CONSUMER_TOKEN
CONSUMER_SECRET
ACCESS_TOKEN
ACCESS_SECRET
"""

Procedure for BigsDB (example with bordetella_1-cgMLST_genus-1415-BIGSdb):

Expand All @@ -254,8 +248,14 @@ After authorizing, it will give you a verification code that you need to enter i

It will then return your access token and secret.

5. Provide the consumer token, consumer secret, access token, and access secret in the data manager tool.
5. Provide the consumer token, consumer secret, access token, and access secret in the data manager tool by setting this bash variables :

"""
CONSUMER_TOKEN
CONSUMER_SECRET
ACCESS_TOKEN
ACCESS_SECRET
"""
]]></help>
<citations>
<citation type="doi">10.3390/microorganisms10020292</citation>
Expand Down
5 changes: 2 additions & 3 deletions data_managers/data_manager_build_coreprofiler/test-data/coreprofiler_scheme.loc.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,5 @@
#
#For example
#coreprofiler_downloaded_20250625_klebsiella_3_scgMLST634_632_loci_bigsdb klebsiella_3: scgMLST634 [632 loci] (BIGSdb) coreprofiler_klebsiella_3 coreprofiler_klebsiella_3/db_klebsiella_3/klebsiella_3.fasta coreprofiler_klebsiella_3/scheme_klebsiella_3
coreprofiler_downloaded_06102025-borrelia_3-cgMLST-639-pubmlst borrelia_3: cgMLST [639 loci] (pubmlst) /tmp/tmpmxz9a8ul/galaxy-dev/tool-data/coreprofiler/coreprofiler_borrelia_3 coreprofiler_borrelia_3/db_borrelia_3/borrelia_3.fasta coreprofiler_borrelia_3/scheme_borrelia_3
coreprofiler_downloaded_06102025-bordetella_1-cgMLST_genus-1415-BIGSdb bordetella_1: cgMLST_genus [1415 loci] (BIGSdb) /tmp/tmpmxz9a8ul/galaxy-dev/tool-data/coreprofiler/coreprofiler_bordetella_1 coreprofiler_bordetella_1/db_bordetella_1/bordetella_1.fasta coreprofiler_bordetella_1/scheme_bordetella_1
coreprofiler_downloaded_06102025-escherichia_v1-cgMLST-2513-enterobase escherichia_v1: cgMLST [2513 loci] (enterobase) /tmp/tmpmxz9a8ul/galaxy-dev/tool-data/coreprofiler/coreprofiler_escherichia_v1 coreprofiler_escherichia_v1/db_escherichia_v1/escherichia_v1.fasta coreprofiler_escherichia_v1/scheme_escherichia_v1
coreprofiler_downloaded_10102025-borrelia_3-cgMLST-639-pubmlst-no_token borrelia_3: cgMLST [639 loci] (pubmlst-no_token) /tmp/tmphn3k3ekk/galaxy-dev/tool-data/coreprofiler/coreprofiler_borrelia_3_no_token coreprofiler_borrelia_3_no_token/db_borrelia_3/borrelia_3.fasta coreprofiler_borrelia_3_no_token/scheme_borrelia_3
coreprofiler_downloaded_10102025-bordetella_1-cgMLST_genus-1415-BIGSdb-no_token bordetella_1: cgMLST_genus [1415 loci] (BIGSdb-no_token) /tmp/tmphn3k3ekk/galaxy-dev/tool-data/coreprofiler/coreprofiler_bordetella_1_no_token coreprofiler_bordetella_1_no_token/db_bordetella_1/bordetella_1.fasta coreprofiler_bordetella_1_no_token/scheme_bordetella_1