Merge pull request #31 from nextflow-io/secrets

Fix default credentials being used if secrets aren't resolved

Author: edmundmiller

README.md

@@ -57,6 +57,8 @@ The following options are available:
 `sql.db.'<DB-NAME>'.password`
 : The database connection password.
 
+For information on using secrets with database credentials, see [docs/secrets.md](docs/secrets.md).
+
 ## Dataflow Operators
 
 This plugin provides the following dataflow operators for querying from and inserting into database tables.

changelog.txt

@@ -1,5 +1,10 @@
 NF-SQLDB CHANGE-LOG
 ===================
+0.7.1 - 21 Aug 2025
+- Fix unresolved secrets detection to prevent silent fallback to default credentials
+- Add comprehensive secrets documentation and troubleshooting guide
+- Improve error messages for unknown database configurations
+
 0.7.0 - 28 May 2025
 - Add sqlExecutor and other minor improvements [072ae039]
 

docs/secrets.md

@@ -0,0 +1,55 @@
+# Using Secrets for Database Credentials
+
+For production deployments, it's recommended to use Nextflow secrets instead of hardcoding credentials in configuration files. This is especially important when working with cloud databases like AWS Athena.
+
+## Configuration with Secrets
+
+When using [Nextflow secrets](https://www.nextflow.io/docs/latest/secrets.html) (available in Nextflow 25.04.0+), you can reference workspace or user-level secrets in your database configuration:
+
+```groovy
+sql {
+    db {
+        athena {
+            url = 'jdbc:awsathena://AwsRegion=us-east-1;S3OutputLocation=s3://bucket;Workgroup=workgroup'
+            user = secrets.ATHENA_USER
+            password = secrets.ATHENA_PASSWORD
+            driver = 'com.simba.athena.jdbc.Driver'
+        }
+    }
+}
+```
+
+## Setting Up Secrets in Seqera Platform
+
+1. **Workspace Secrets**: Navigate to your workspace → Secrets → Add secret
+2. **User Secrets**: Navigate to Your profile → Secrets → Add secret  
+3. Create secrets with names matching your configuration (e.g., `ATHENA_USER`, `ATHENA_PASSWORD`)
+
+## Troubleshooting Secrets Issues
+
+If you encounter authentication errors like "Missing credentials error", verify:
+
+- **Secret Names**: Ensure secret names in your configuration match exactly (case-sensitive)
+- **Permissions**: Verify you have access to workspace secrets or have defined user-level secrets
+- **Nextflow Version**: Secrets require Nextflow >=25.04.0
+- **Secret Values**: Ensure secrets contain valid credentials (no empty values)
+
+Common error patterns:
+- `user=sa; password=null` indicates secrets were not resolved
+- `Unresolved secret detected` means secret names don't match or aren't accessible
+
+## Local Development
+
+For local testing, you can use the Nextflow secrets command:
+
+```bash
+# Set secrets locally
+nextflow secrets set ATHENA_USER "your-username"  
+nextflow secrets set ATHENA_PASSWORD "your-password"
+
+# List secrets
+nextflow secrets list
+
+# Run pipeline with secrets
+nextflow run your-pipeline.nf
+```

plugins/nf-sqldb/src/main/nextflow/sql/ChannelSqlExtension.groovy

@@ -106,11 +106,9 @@ class ChannelSqlExtension extends PluginExtensionPoint {
         final dataSource = config.getDataSource(dsName)
         if( dataSource==null ) {
             def msg = "Unknown db name: $dsName"
-            def choices = config.getDataSourceNames().closest(dsName) ?: config.getDataSourceNames()
-            if( choices?.size() == 1 )
-                msg += " - Did you mean: ${choices.get(0)}?"
-            else if( choices )
-                msg += " - Did you mean any of these?\n" + choices.collect { "  $it"}.join('\n') + '\n'
+            def choices = config.getDataSourceNames()
+            if( choices )
+                msg += " - Available databases: " + choices.join(', ')
             throw new IllegalArgumentException(msg)
         }
         return dataSource

plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy

@@ -43,21 +43,53 @@ class SqlDataSource {
     SqlDataSource(Map opts) {
         this.url = opts.url ?: DEFAULT_URL
         this.driver = opts.driver ?: urlToDriver(url) ?: DEFAULT_DRIVER
-        this.user = opts.user ?: DEFAULT_USER
-        this.password = opts.password
+        this.user = resolveCredential(opts.user, 'user') ?: DEFAULT_USER
+        this.password = resolveCredential(opts.password, 'password')
     }
 
     SqlDataSource(Map opts, SqlDataSource fallback) {
         this.url = opts.url ?: fallback.url ?: DEFAULT_URL
         this.driver = opts.driver ?: urlToDriver(url) ?: fallback.driver ?: DEFAULT_DRIVER
-        this.user = opts.user ?: fallback.user ?: DEFAULT_USER
-        this.password = opts.password ?: fallback.password
+        this.user = resolveCredential(opts.user, 'user') ?: fallback.user ?: DEFAULT_USER
+        this.password = resolveCredential(opts.password, 'password') ?: fallback.password
     }
 
     protected String urlToDriver(String url) {
         DriverRegistry.DEFAULT.urlToDriver(url)
     }
 
+    /**
+     * Resolves a credential value, checking for unresolved secrets and providing appropriate error handling
+     * 
+     * @param value The credential value from configuration
+     * @param credType The type of credential ('user' or 'password') for error messages
+     * @return The resolved credential value, or null if not provided
+     * @throws IllegalArgumentException if an unresolved secret is detected
+     */
+    protected String resolveCredential(Object value, String credType) {
+        if (value == null) {
+            return null
+        }
+        
+        String stringValue = value.toString()
+        
+        // Check for unresolved secrets (patterns like 'secrets.ATHENA_USER' or similar)
+        if (stringValue.startsWith('secrets.') || stringValue.contains('secret') && stringValue.contains('[') && stringValue.contains(']')) {
+            throw new IllegalArgumentException(
+                "Unresolved secret detected for $credType: '$stringValue'. " +
+                "This typically indicates that workspace secrets are not properly configured or accessible. " +
+                "Please verify that:\n" +
+                "1. The secret is defined in your workspace/user secrets\n" +
+                "2. The secret name matches exactly (case-sensitive)\n" +
+                "3. You have proper permissions to access the secret\n" +
+                "4. The Nextflow version supports secrets integration (>=25.04.0)\n" +
+                "See: https://www.nextflow.io/docs/latest/secrets.html"
+            )
+        }
+        
+        return stringValue.isEmpty() ? null : stringValue
+    }
+
     Map toMap() {
         final result = new HashMap(10)
         if( url )

plugins/nf-sqldb/src/test/nextflow/sql/ChannelSqlExtensionTest.groovy

@@ -95,4 +95,34 @@ class ChannelSqlExtensionTest extends Specification {
         rows.alpha == ['x1','y2','z3']
     }
 
+    def 'should error on unresolved secrets' () {
+        given:
+        def session = Mock(Session) {
+            getConfig() >> [sql: [db: [test: [user: 'secrets.ATHENA_USER']]]]
+        }
+
+        when:
+        new ChannelSqlExtension().init(session)
+
+        then:
+        thrown(IllegalArgumentException)
+    }
+
+    def 'should handle unknown database' () {
+        given:
+        def session = Mock(Session) {
+            getConfig() >> [sql: [db: [default: [url: 'jdbc:h2:mem:'], postgres: [url: 'jdbc:postgresql:']]]]
+        }
+        def sqlExtension = new ChannelSqlExtension()
+        sqlExtension.init(session)
+
+        when:
+        sqlExtension.fromQuery([db: 'invalid'], 'select * from table')
+
+        then:
+        def e = thrown(IllegalArgumentException)
+        e.message.contains("Unknown db name: invalid")
+        e.message.contains("Available databases:")
+    }
+
 }

plugins/nf-sqldb/src/test/nextflow/sql/SqlDslTest.groovy

@@ -209,6 +209,12 @@ class SqlDslTest extends Dsl2Spec {
     @Requires({System.getenv('NF_SQLDB_TEST_ATHENA_REGION')})
     @Requires({System.getenv('NF_SQLDB_ATHENA_TEST_S3_BUCKET')})
     @IgnoreIf({ System.getenv('NXF_SMOKE') })
+    @Requires({
+        System.getenv('NF_SQLDB_TEST_ATHENA_USERNAME') &&
+        System.getenv('NF_SQLDB_TEST_ATHENA_PASSWORD') &&
+        System.getenv('NF_SQLDB_TEST_ATHENA_REGION') &&
+        System.getenv('NF_SQLDB_ATHENA_TEST_S3_BUCKET')
+    })
     @Timeout(60)
     def 'should perform a query for AWS Athena and create a channel'() {
         given:

plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy

@@ -132,4 +132,30 @@ class SqlDataSourceTest extends Specification {
         ds1.hashCode() == ds2.hashCode()
         ds1.hashCode() != ds3.hashCode()
     }
+
+    def 'should detect unresolved secrets' () {
+        when:
+        new SqlDataSource([user: pattern])
+        then:
+        def e = thrown(IllegalArgumentException)
+        e.message.contains("Unresolved secret detected")
+        e.message.contains("workspace secrets are not properly configured")
+
+        where:
+        pattern << ['secrets.ATHENA_USER', '[secret]']
+    }
+
+    def 'should handle various credential inputs' () {
+        when:
+        def ds = new SqlDataSource([user: userInput, password: passInput])
+        then:
+        ds.user == expectedUser
+        ds.password == expectedPass
+
+        where:
+        userInput    | passInput    | expectedUser              | expectedPass
+        'validuser'  | 'validpass'  | 'validuser'              | 'validpass'
+        null         | null         | SqlDataSource.DEFAULT_USER| null
+        ''           | ''           | SqlDataSource.DEFAULT_USER| null
+    }
 }