tessl/pypi-azure-storage-file-datalake

Microsoft Azure File DataLake Storage Client Library for Python

Overall
score

92%

Overview

Eval results

Files

File System Operations

Name: tessl/pypi-azure-storage-file-datalake
Author: tessl

File system-level operations for managing directories, files, and access policies within a specific container. The FileSystemClient provides comprehensive management capabilities for hierarchical namespace operations.

Capabilities

FileSystemClient

Client to interact with a specific file system, providing operations for creating and managing directories and files within the file system scope.

class FileSystemClient:
    """
    A client to interact with a specific Azure Data Lake Storage Gen2 file system.
    
    Attributes:
        url (str): The full endpoint URL to the file system, including SAS token if used
        primary_endpoint (str): The full primary endpoint URL
        primary_hostname (str): The hostname of the primary endpoint
        file_system_name (str): Name of the file system
    """
    
    def __init__(
        self,
        account_url: str,
        file_system_name: str,
        credential=None,
        **kwargs
    ):
        """
        Initialize the FileSystemClient.
        
        Args:
            account_url (str): The URL to the DataLake storage account
            file_system_name (str): Name of the file system
            credential: Authentication credential
            **kwargs: Additional client configuration options
        """
    
    @classmethod
    def from_connection_string(
        cls,
        conn_str: str,
        file_system_name: str,
        credential=None,
        **kwargs
    ) -> 'FileSystemClient':
        """
        Create FileSystemClient from connection string.
        
        Args:
            conn_str (str): Connection string for the storage account
            file_system_name (str): Name of the file system
            credential: Optional credential to override connection string auth
            **kwargs: Additional client configuration options
            
        Returns:
            FileSystemClient: The file system client instance
        """
    
    def close(self) -> None:
        """Close the client."""
        
    def __exit__(self, *args) -> None:
        """Context manager exit."""

Usage Examples:

from azure.storage.filedatalake import FileSystemClient

# Create client directly
fs_client = FileSystemClient(
    account_url="https://mystorageaccount.dfs.core.windows.net",
    file_system_name="myfilesystem",
    credential="<account_key>"
)

# From connection string
fs_client = FileSystemClient.from_connection_string(
    "DefaultEndpointsProtocol=https;AccountName=mystorageaccount;AccountKey=<key>",
    file_system_name="myfilesystem"
)

File System Management

Operations for creating, deleting, and managing the file system itself.

def create_file_system(
    self,
    metadata: Dict[str, str] = None,
    public_access: PublicAccess = None,
    **kwargs
) -> Dict[str, Any]:
    """
    Create the file system.
    
    Args:
        metadata (dict, optional): Metadata key-value pairs
        public_access (PublicAccess, optional): Public access level
        **kwargs: Additional options
        
    Returns:
        dict: File system creation response headers
    """

def delete_file_system(self, **kwargs) -> None:
    """
    Delete the file system.
    
    Args:
        **kwargs: Additional options including lease conditions
    """

def exists(self, **kwargs) -> bool:
    """
    Check if the file system exists.
    
    Args:
        **kwargs: Additional options
        
    Returns:
        bool: True if file system exists, False otherwise
    """

def get_file_system_properties(self, **kwargs) -> FileSystemProperties:
    """
    Get file system properties.
    
    Args:
        **kwargs: Additional options including lease conditions
        
    Returns:
        FileSystemProperties: Properties of the file system
    """

def set_file_system_metadata(
    self,
    metadata: Dict[str, str],
    **kwargs
) -> Dict[str, Any]:
    """
    Set file system metadata.
    
    Args:
        metadata (dict): Metadata key-value pairs
        **kwargs: Additional options including conditions
        
    Returns:
        dict: Response headers
    """

Access Policy Management

Operations for managing signed access policies and public access settings.

def set_file_system_access_policy(
    self,
    signed_identifiers: Dict[str, AccessPolicy],
    public_access: PublicAccess = None,
    **kwargs
) -> Dict[str, Any]:
    """
    Set access policy for the file system.
    
    Args:
        signed_identifiers (dict): Signed access policies
        public_access (PublicAccess, optional): Public access level
        **kwargs: Additional options
        
    Returns:
        dict: Response headers
    """

def get_file_system_access_policy(self, **kwargs) -> Dict[str, Any]:
    """
    Get access policy for the file system.
    
    Args:
        **kwargs: Additional options
        
    Returns:
        dict: Access policy information including public access and signed identifiers
    """

Directory Operations

Operations for creating, deleting, and managing directories within the file system.

def create_directory(
    self,
    directory: Union[DirectoryProperties, str],
    metadata: Dict[str, str] = None,
    **kwargs
) -> DataLakeDirectoryClient:
    """
    Create a directory in the file system.
    
    Args:
        directory: Name of the directory or DirectoryProperties object
        metadata (dict, optional): Metadata key-value pairs
        **kwargs: Additional options including permissions and conditions
        
    Returns:
        DataLakeDirectoryClient: Client for the created directory
    """

def delete_directory(
    self,
    directory: Union[DirectoryProperties, str],
    **kwargs
) -> DataLakeDirectoryClient:
    """
    Delete a directory from the file system.
    
    Args:
        directory: Name of the directory or DirectoryProperties object
        **kwargs: Additional options including recursive delete and conditions
        
    Returns:
        DataLakeDirectoryClient: Client for the deleted directory
    """

def get_directory_client(
    self,
    directory: Union[DirectoryProperties, str]
) -> DataLakeDirectoryClient:
    """
    Get a DataLakeDirectoryClient for a specific directory.
    
    Args:
        directory: Name of the directory or DirectoryProperties object
        
    Returns:
        DataLakeDirectoryClient: Client for the specified directory
    """

File Operations

Operations for creating, deleting, and managing files within the file system.

def create_file(
    self,
    file: Union[FileProperties, str],
    **kwargs
) -> DataLakeFileClient:
    """
    Create a file in the file system.
    
    Args:
        file: Name/path of the file or FileProperties object
        **kwargs: Additional options including content settings, metadata, permissions
        
    Returns:
        DataLakeFileClient: Client for the created file
    """

def delete_file(
    self,
    file_path: Union[FileProperties, str],
    **kwargs
) -> DataLakeFileClient:
    """
    Delete a file from the file system.
    
    Args:
        file_path: Path to the file or FileProperties object
        **kwargs: Additional options including conditions
        
    Returns:
        DataLakeFileClient: Client for the deleted file
    """

def get_file_client(
    self,
    file_path: Union[FileProperties, str]
) -> DataLakeFileClient:
    """
    Get a DataLakeFileClient for a specific file.
    
    Args:
        file_path: Path to the file or FileProperties object
        
    Returns:
        DataLakeFileClient: Client for the specified file
    """

Path Listing and Discovery

Operations for listing and discovering paths within the file system hierarchy.

def get_paths(
    self,
    path: str = None,
    recursive: bool = True,
    max_results: int = None,
    **kwargs
) -> ItemPaged[PathProperties]:
    """
    List paths in the file system.
    
    Args:
        path (str, optional): Path prefix to filter results
        recursive (bool): Whether to list recursively (default: True)
        max_results (int, optional): Maximum number of results per page
        **kwargs: Additional options including upn (user principal names)
        
    Returns:
        ItemPaged[PathProperties]: Paged list of path properties
    """

def list_deleted_paths(self, **kwargs) -> ItemPaged[DeletedPathProperties]:
    """
    List deleted paths that can be restored.
    
    Args:
        **kwargs: Additional options including path_prefix and max_results
        
    Returns:
        ItemPaged[DeletedPathProperties]: Paged list of deleted path properties
    """

Lease Management

Operations for acquiring and managing leases on the file system.

def acquire_lease(
    self,
    lease_duration: int = -1,
    lease_id: str = None,
    **kwargs
) -> DataLakeLeaseClient:
    """
    Acquire a lease on the file system.
    
    Args:
        lease_duration (int): Duration of the lease in seconds (-1 for infinite)
        lease_id (str, optional): Proposed lease ID
        **kwargs: Additional options including conditions
        
    Returns:
        DataLakeLeaseClient: Lease client for managing the lease
    """

Usage Examples:

from azure.storage.filedatalake import FileSystemClient, PublicAccess

# Create a file system client
fs_client = FileSystemClient(
    account_url="https://mystorageaccount.dfs.core.windows.net",
    file_system_name="myfilesystem",
    credential="<account_key>"
)

# Create the file system if it doesn't exist
if not fs_client.exists():
    fs_client.create_file_system(
        metadata={"purpose": "data-processing"},
        public_access=PublicAccess.FileSystem
    )

# List all paths in the file system
paths = fs_client.get_paths(recursive=True)
for path in paths:
    print(f"Path: {path.name}, Type: {'Directory' if path.is_directory else 'File'}")

# Create a directory structure
analytics_dir = fs_client.create_directory("analytics")
raw_data_dir = fs_client.create_directory("analytics/raw-data")

# Create a file
data_file = fs_client.create_file("analytics/processed/results.json")

# Acquire a lease for exclusive access
lease_client = fs_client.acquire_lease(lease_duration=30)
print(f"Acquired lease: {lease_client.id}")

Install with Tessl CLI

npx tessl i tessl/pypi-azure-storage-file-datalake

tessl/pypi-azure-storage-file-datalake