Allow lakeFSFS to work with Azure (#5986)

treeverse · Jun 1, 2023 · 8d96ef0 · 8d96ef0
1 parent fbf2d49
commit 8d96ef0
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java b/clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java
@@ -86,6 +86,7 @@ public class LakeFSFileSystem extends FileSystem {
     private boolean failedFSForConfig = false;
     private PhysicalAddressTranslator physicalAddressTranslator;
     private StorageAccessStrategy storageAccessStrategy;
+    private AccessMode accessMode;
     private static File emptyFile = new File("/dev/null");
 
     // Currently bulk deletes *must* receive a single-threaded executor!
@@ -136,7 +137,7 @@ void initializeWithClientFactory(URI name, Configuration conf, ClientFactory cli
             throw new IOException("Failed to get lakeFS blockstore type", e);
         }
         String accessModeConf = FSConfiguration.get(conf, uri.getScheme(), ACCESS_MODE_KEY_SUFFIX);
-        AccessMode accessMode = AccessMode.valueOf(StringUtils.defaultIfBlank(accessModeConf, AccessMode.SIMPLE.toString()).toUpperCase());
+        accessMode = AccessMode.valueOf(StringUtils.defaultIfBlank(accessModeConf, AccessMode.SIMPLE.toString()).toUpperCase());
         if (accessMode == AccessMode.PRESIGNED) {
             storageAccessStrategy = new PresignedStorageAccessStrategy(this, lfsClient);
         } else if (accessMode == AccessMode.SIMPLE) {
@@ -150,7 +151,7 @@ private synchronized FileSystem getFSForConfig() {
         if (fsForConfig != null) {
             return fsForConfig;
         }
-        if (failedFSForConfig) {
+        if (failedFSForConfig || accessMode == AccessMode.PRESIGNED) {
             return null;
         }
         Path path = new Path(uri);

diff --git a/docs/integrations/spark.md b/docs/integrations/spark.md
@@ -682,7 +682,7 @@ and then add these into a configuration file, e.g., `$SPARK_HOME/conf/hdfs-site.
 Add the following the cluster's configuration under `Configuration ➡️ Advanced options`:
 
 ```
-spark.hadoop.fs.access.mode presigned
+spark.hadoop.fs.lakefs.access.mode presigned
 spark.hadoop.fs.lakefs.impl io.lakefs.LakeFSFileSystem
 spark.hadoop.fs.lakefs.access.key AKIAlakefs12345EXAMPLE
 spark.hadoop.fs.lakefs.secret.key abc/lakefs/1234567bPxRfiCYEXAMPLEKEY