API Reference

Table Classes

Abstract Base

An SDMLTable. This is the abstract superclass for all Simple Data Markup Language tables, and implements the schema methods of every SDML class. The data methods are implemented by the concrete classes. Any new SDMLTable class should: 1. Subclass SDMLTable 2. Have a constructor with the argument schema 3. call super(<classname, self).init(schema) in the constructor 4. Implement the methods: (a) all_values(self, column_name) (b) range_spec(self, column_nam) (c) _get_filtered_rows_from_filter(self, filter, columns = None) (d) to_json(self) where: i. column_name is the name of the column to get the values/range_spec from ii. filter is a an instance of SDQLFilter iii. if columns is not None for get_filtered_rows, only return entries from those columns in the result from get_filtered_rows Arguments: schema: a list of records of the form {"name": <column_name, "type": }. The column_type must be a type from galyleo_constants.SDTP_TYPES.

Source code in src/sdtp/sdtp_table.py

def __init__(self, schema):
    if type(schema) != list:
        raise InvalidDataException(f'The schema must be a list of dictionaries, not {type(schema)}')
    error_entries = [get_errors(entry) for entry in schema]
    error_entries = [entry for entry in error_entries if len(entry) > 0]
    if len(error_entries) > 0:
        raise InvalidDataException(f"Errors in schema {schema}: {error_entries}")

    self.schema = schema

all_values(column_name)

get all the values from column_name Arguments: column_name: name of the column to get the values for

Returns:

Name	Type	Description
list	list	List of the values

Source code in src/sdtp/sdtp_table.py

def all_values(self, column_name: str)  -> list:
    '''
    get all the values from column_name
    Arguments:
        column_name: name of the column to get the values for

    Returns:
        list: List of the values

    '''
    raise InvalidDataException(f'all_values has not been in {type(self)}.__name__')

column_names()

Return the names of the columns

Source code in src/sdtp/sdtp_table.py

def column_names(self):
    '''
    Return the names of the columns
    '''
    return [column["name"] for column in self.schema]

column_types()

Return the types of the columns

Source code in src/sdtp/sdtp_table.py

def column_types(self):
    '''
    Return the types of the columns
    '''
    return [column["type"] for column in self.schema]

get_column(column_name)

get the column column_name Arguments: column_name: name of the column to get

Returns:

Name	Type	Description
list	list	List of the values in the column

Source code in src/sdtp/sdtp_table.py

def get_column(self, column_name: str)  -> list:
    '''
    get the column  column_name
    Arguments:
        column_name: name of the column to get 


    Returns:
        list: List of the values in the column

    '''
    raise InvalidDataException(f'get_column has not been in {type(self)}.__name__')

get_column_type(column_name)

Returns the type of column column_name, or None if this table doesn't have a column with name column_name.

Parameters:

Name	Type	Description	Default
column_name	str	name of the column to get the type for	required

Source code in src/sdtp/sdtp_table.py

def get_column_type(self, column_name):
    '''
    Returns the type of column column_name, or None if this table doesn't have a column with
    name  column_name.

    Arguments:
        column_name(str): name of the column to get the type for
    '''
    matches = [column["type"] for column in self.schema if column["name"] == column_name]
    if len(matches) == 0:
        return None
    else:
        return matches[0]

get_filtered_rows(filter_spec=None, columns=None, format=DEFAULT_FILTERED_ROW_RESULT_FORMAT)

Filter the rows according to the specific-ation given by filter_spec. Returns the rows for which the resulting filter returns True.

Parameters:

Name	Type	Description	Default
filter_spec	dict	Specification of the filter, as a dictionary	None
columns	list	the names of the columns to return. Returns all columns if absent	None
format	str	one of 'list', 'dict', 'sdml'. Default is list.	DEFAULT_FILTERED_ROW_RESULT_FORMAT

Returns: list: The subset of self.get_rows() which pass the filter in the format specified by format

Source code in src/sdtp/sdtp_table.py

def get_filtered_rows(self, filter_spec=None, columns=None, format = DEFAULT_FILTERED_ROW_RESULT_FORMAT):
    '''
    Filter the rows according to the specific-ation given by filter_spec.
    Returns the rows for which the resulting filter returns True.

    Arguments:
        filter_spec(dict): Specification of the filter, as a dictionary
        columns(list): the names of the columns to return.  Returns all columns if absent
        format(str): one of 'list', 'dict', 'sdml'.  Default is list.  
    Returns:
        list: The subset of self.get_rows() which pass the filter in the format specified by format
    '''
    # Check to make sure that the format is valid
    if format is None: format = DEFAULT_FILTERED_ROW_RESULT_FORMAT

    if format not in ALLOWED_FILTERED_ROW_RESULT_FORMATS:
        raise InvalidDataException(f'format for get_filtered rows must be one of {ALLOWED_FILTERED_ROW_RESULT_FORMATS}, not {format}')
    requested_columns = columns  if columns else []
    # Note that we don't check if the column names are all valid
    filter = make_filter(filter_spec) if filter_spec is not None else None
    rows =  self._get_filtered_rows_from_filter(filter = filter)
    columns_in_result = self.column_names() if len(requested_columns) == 0 else requested_columns
    return _convert_filter_result_to_format(rows, columns_in_result, self.schema, format)

range_spec(column_name)

Get the dictionary {min_val, max_val} for column_name Arguments:

column_name: name of the column to get the range spec for

Returns:

Name	Type	Description
list	list	the minimum and maximum of the column

Source code in src/sdtp/sdtp_table.py

def range_spec(self, column_name: str) -> list:
    '''
    Get the dictionary {min_val, max_val} for column_name
    Arguments:

        column_name: name of the column to get the range spec for

    Returns:
        list: the minimum and  maximum of the column

    '''
    raise InvalidDataException(f'range_spec has not been in {type(self)}.__name__')

to_dictionary()

Return the dictionary of this table, for saving on disk or transmission.

Source code in src/sdtp/sdtp_table.py

def to_dictionary(self):
    '''
    Return the dictionary  of this table, for saving on disk or transmission.
    '''
    raise InvalidDataException(f'to_dictionary has not been in {type(self)}.__name__')

to_json()

Return the JSON form of this table, for saving on disk or transmission.

Source code in src/sdtp/sdtp_table.py

def to_json(self):
    '''
    Return the JSON form of this table, for saving on disk or transmission.
    '''
    # Since the columns are already a dictionary, they are simply directly jsonified.  For the rows,
    # just use jjson.dumps, making sure to convert types appropriately

    return json.dumps(self.to_dictionary(), default = json_serialize, indent = 2)

Concrete Table Types

Bases: SDMLTable

A SDMLFixedTable: This is a convenience class for subclasses which generate a fixed number of rows locally, independent of filtering. This is instantiated with a function get_rows() which delivers the rows, rather than having them explicitly in the Table. Note that get_rows() must return a list of rows, each of which has the appropriate number of entries of the appropriate types. all_values, range_spec, and _get_filtered_rows_from_filter are all implemented on top of get_rows. Note that these methods can be overridden in a subclass if there is a more efficient method than the obvious implementation, which is what's implemented here.

Parameters:

Name	Type	Description	Default
schema	List	a list of records of the form {"name": <column_name, "type": }. The column_type must be a type from galyleo_constants.SDTP_TYPES.	required
get_rows	Callable	a function which returns a list of list of values. Each component list must have the same length as schema, and the jth element must be of the type specified in the jth element of schema	required

Source code in src/sdtp/sdtp_table.py

def __init__(self, schema: list, get_rows: Callable[[], List]):
    super(SDMLFixedTable, self).__init__(schema)
    self.get_rows = get_rows

all_values(column_name)

get all the values from column_name Arguments: column_name: name of the column to get the values for

Returns:

Name	Type	Description
list	list	List of the values

Source code in src/sdtp/sdtp_table.py

def all_values(self, column_name: str) -> list:
    '''
    get all the values from column_name
    Arguments:
        column_name: name of the column to get the values for


    Returns:
        list: List of the values

    '''
    (values, sdtp_type) = self._get_column_values_and_type(column_name)
    result = list(set(values))
    result.sort()
    return result

check_column_type(column_name)

For testing. Makes sure that all the entries in column_name are the right type No return, but throws an InvalidDataException if there's a bad element in the column

Source code in src/sdtp/sdtp_table.py

def check_column_type(self, column_name):
    '''
    For testing.  Makes sure that all the entries in column_name are the right type
    No return, but throws an InvalidDataException if there's a bad element in the column
    '''
    value_list = self.all_values(column_name)
    required_type = self.get_column_type(column_name)
    if required_type is not None:
        bad_values = [val for val in value_list if not type_check(required_type, val)]
    else:
        bad_values = []
    if len(bad_values) > 0:
        raise InvalidDataException(f'Values {bad_values} could not be converted to {required_type} in column {column_name}')

get_column(column_name)

get all the column column_name Arguments: column_name: name of the column to get

Returns:

Name	Type	Description
list	list	The column as a list

Source code in src/sdtp/sdtp_table.py

def get_column(self, column_name: str)  -> list:
    '''
    get all the column  column_name
    Arguments:
        column_name: name of the column to get


    Returns:
        list: The column as a list

    '''
    (result, sdtp_type) = self._get_column_values_and_type(column_name)
    return  result

range_spec(column_name)

Get the dictionary {min_val, max_val} for column_name Arguments: column_name: name of the column to get the range spec for

Returns:

Name	Type	Description
list	list	the minimum and maximum of the column

Source code in src/sdtp/sdtp_table.py

def range_spec(self, column_name: str) -> list:
    '''
    Get the dictionary {min_val, max_val} for column_name
    Arguments:
        column_name: name of the column to get the range spec for

    Returns:
        list: the minimum and  maximum of the column

    '''
    (values, sdtp_type) = self._get_column_values_and_type(column_name)
    values.sort()
    result = [values[0], values[-1]]
    return result

to_dataframe()

Convert the table to a PANDAS DataFrame. This is very straightforward; just use get_rows to get the rows and convert the schema to the appropriate dtypes. Note this relies on PANDAS type inference.

Source code in src/sdtp/sdtp_table.py

def to_dataframe(self):
    '''
    Convert the table to a PANDAS DataFrame.  This is very straightforward; just 
    use get_rows to get the rows and convert the schema to the appropriate dtypes.
    Note this relies on PANDAS type inference.
    '''

    return  pd.DataFrame(self.get_rows(), columns = self.column_names())

to_dictionary()

Return the intermediate form of this table as a dictioary

Source code in src/sdtp/sdtp_table.py

def to_dictionary(self):
    '''
    Return the intermediate form of this table as a dictioary
    '''
    return {
        "type": "RowTable",
        "schema": self.schema,
        "rows": jsonifiable_rows(self.get_rows(), self.column_types())
    }

Bases: SDMLFixedTable

A simple utility class to serve data from a static list of rows, which can be constructed from a CSV file, Excel File, etc. The idea is to make it easy for users to create and upload simple datasets to be served from a general-purpose server. Note that this will need some authentication.

Source code in src/sdtp/sdtp_table.py

def __init__(self, schema, rows):
    self.schema = schema  # set this *before* calling self.column_types()
    type_list = self.column_types()
    # print("RowTable.__init__ received schema:", schema)
    # print("RowTable.__init__ received rows:", rows)
    # print("RowTable.__init__ inferred column types:", self.column_types(), type_list)
    self.rows = convert_rows_to_type_list(type_list, rows)
    super(RowTable, self).__init__(schema, self._get_rows)

Bases: SDMLTable

A SDTP Table on a remote server. This just has a schema, an URL, and header variables. This is the primary class for the client side of the SDTP, and in many packages would be a separate client module. However, the SDTP is designed so that Remote Tables can be used to serve local tables, so this is part of a server-side framework to. Parameters: table_name: name of the resmote stable schema: schema of the remote table url: url of the server hosting the remore table auth: dictionary of variables and values required to access the table Throws: InvalidDataException if the table doesn't exist on the server, the url is unreachable, the schema doesn't match the downloaded schema

Source code in src/sdtp/sdtp_table.py

def __init__(self, table_name, schema, url,  auth = None, header_dict  = None): 
    super(RemoteSDMLTable, self).__init__(schema)
    self.url = url
    self.schema: List[ColumnSpec] = schema
    self.table_name = table_name
    self.auth = auth
    self.ok = False
    self.header_dict = header_dict
    self.headers = _make_headers(auth, header_dict)

all_values(column_name)

get all the values from column_name Arguments:

column_name: name of the column to get the values for

Returns:

Name	Type	Description
list	list	List of the values

Source code in src/sdtp/sdtp_table.py

def all_values(self, column_name: str) -> list:
    '''
    get all the values from column_name
    Arguments:

        column_name: name of the column to get the values for

    Returns:
        list: List of the values

    '''
    return self._execute_column_route(column_name,  'get_all_values')

connect_with_server()

Connect with the server, ensuring that the server is: a. a SDTP server b. has self.table_name in its list of tables c. the table there has a matching schema

Source code in src/sdtp/sdtp_table.py

def connect_with_server(self):
    '''
    Connect with the server, ensuring that the server is:
    a. a SDTP server
    b. has self.table_name in its list of tables
    c. the table there has a matching schema
    '''

    try:
        response = self._execute_get_request(f'{self.url}/get_tables')
        if response.status_code >= 300:
            self._connect_error(f'Bad connection with {self.url}: code {response.status_code}')
    except Exception as e:
        self._connect_error(f'Error connecting with {self.url}/get_tables: {repr(e)}')
    try:
        server_tables = response.json()
    except Exception as e:
        self._connect_error(f'Error {repr(e)} reading tables from  {self.url}/get_tables')
    if self.table_name in server_tables:
        server_schema: List[ColumnSpec] = server_tables[self.table_name]
        self._check_schema_match(server_schema)
        self.server_schema = server_schema
    else:
        self._connect_error(f'Server at {self.url} does not have table {self.table_name}')
    # if we get here, everything worked:
    self.ok = True

get_column(column_name)

get the column column_name Arguments: column_name: name of the column to get

Returns:

Name	Type	Description
List	list	The column as a list

Source code in src/sdtp/sdtp_table.py

def get_column(self, column_name: str)  -> list:
    '''
    get the column  column_name
    Arguments:
        column_name: name of the column to get


    Returns:
        List: The column as a list
    '''
    return self._execute_column_route(column_name, 'get_column')

get_filtered_rows(filter_spec=None, columns=None, format=DEFAULT_FILTERED_ROW_RESULT_FORMAT)

Filter the rows according to the specification given by filter_spec. Returns the rows for which the resulting filter returns True. Reorders columns to match client request, even if remote server responds in different order. This guarantees protocol safety.

Parameters:

Name	Type	Description	Default
filter_spec	dict	Specification of the filter, as a dictionary	None
columns	list	the names of the columns to return. Returns all columns if absent	None
format	str	one of 'list', 'dict', 'sdml'. Default is list.	DEFAULT_FILTERED_ROW_RESULT_FORMAT

Returns: The subset of self.get_rows() which pass the filter in the format specified by format

Source code in src/sdtp/sdtp_table.py

def get_filtered_rows(self, filter_spec=None, columns=None, format = DEFAULT_FILTERED_ROW_RESULT_FORMAT) -> Union[list, list[dict[str, Any]], RowTable]:
    '''
    Filter the rows according to the specification given by filter_spec.
    Returns the rows for which the resulting filter returns True.
    Reorders columns to match client request, even if remote server responds in different order. This guarantees protocol safety.

    Arguments:
        filter_spec (dict): Specification of the filter, as a dictionary
        columns (list): the names of the columns to return.  Returns all columns if absent
        format (str): one of 'list', 'dict', 'sdml'.  Default is list.  
    Returns:
        The subset of self.get_rows() which pass the filter in the format specified by format
    '''
    # Check to make sure that the format is valid
    if format is None: format = DEFAULT_FILTERED_ROW_RESULT_FORMAT
    requested_columns = columns if columns else []


    if format not in ALLOWED_FILTERED_ROW_RESULT_FORMATS:
        raise InvalidDataException(f'format for get_filtered rows must be one of {ALLOWED_FILTERED_ROW_RESULT_FORMATS}, not {format}')
    # Note that we don't check if the column names are all valid

    remoteRowTable = self._get_filtered_rows_from_remote(filter_spec, columns=requested_columns)
    if format == 'list':
        column_names = self.column_names() if len(requested_columns) == 0 else requested_columns
        return _generate_ordered_lists(remoteRowTable, self, column_names)
    elif format == 'dict':
        result_columns = remoteRowTable.column_names()
        return [_row_dict(row, result_columns) for row in remoteRowTable.rows]
    else:
        return remoteRowTable

range_spec(column_name)

Get the dictionary {min_val, max_val} for column_name Arguments:

column_name: name of the column to get the range spec for

Returns:

Name	Type	Description
list	list	the minimum and maximum of the column

Source code in src/sdtp/sdtp_table.py

def range_spec(self, column_name: str)  -> list:
    '''
    Get the dictionary {min_val, max_val} for column_name
    Arguments:

        column_name: name of the column to get the range spec for

    Returns:
        list: the minimum and  maximum of the column

    '''
    return self._execute_column_route(column_name, 'get_range_spec')

Table Factories

Abstract Base

Bases: ABC

A class which builds an SDMLTable of a specific type. All SDMLTables have a schema, but after that the specification varies, depending on the method the table uses to get the table rows. Specific factories should subclass this and instantiate the class method build_table. The tag is the table type, simply a string which indicates which class of table should be built. A new SDMLTableFactory class should be built for each concrete subclass of SDMLTable, and ideally in the same file. The SDMLTable subclass should put a "type" field in the intermediate form, and the value of "type" should be the type built by the SDTP Table field SDMLTableFactory is an abstract class -- each concrete subclass should call the init method on the table_type on initialization. build_table is the method which actually builds the table; the superclass convenience version of the method throws an InvalidDataException if the spec has the wrong table type Every subclass should set the table_type. This attribute registers the tables which this Factory builds

check_table_type(table_type) classmethod

Check to make sure the type is right. If not, throw an InvalidDataException

Source code in src/sdtp/sdtp_table_factory.py

@classmethod
def check_table_type(cls, table_type):
  '''
  Check to make sure the type is right.  If not, throw an InvalidDataException
  '''
  factory_class = TableBuilder.get_factory(table_type)
  if cls != factory_class:
      raise InvalidDataException(f"Wrong factory for {table_type}: expected {factory_class}, got {cls}")

Concrete Factories

Bases: SDMLTableFactory

A factory to build RowTables -- in fact, all SDMLFixedTables. build_table is very simple, just instantiating a RowTable on the rows and schema of the specification

Bases: SDMLTableFactory

A factory to build RemoteSDMLTables. build_table is very simple, just instantiating a RemoteSDMLTables on the url and schema of the specification

Table Builder

A global table builder. This will build a table of any type. This has four methods, all class methods: (1) Build a table from a specification (2) register a class to build a table for a type (3) Get the current mapping of table types to classes (4) Get the class for a particular table type

build_table(spec, *args, **kwargs) classmethod

Build a table from a spec. Argments: spec: an SDML table spec args: additional arguments required to build the table, if any *kwargs: additional arguments required to build the table, if any Returns: The SDML Table Raises: Invalid Data Exception if the spec is not well-formed

Source code in src/sdtp/sdtp_table_factory.py

@classmethod
def build_table(cls, spec, *args, **kwargs):
    '''
    Build a table from a spec.
    Argments:
      spec: an SDML table spec
      *args: additional arguments required to build the table, if any
      **kwargs: additional arguments required to build the table, if any
    Returns:
      The SDML Table
    Raises:
      Invalid Data Exception if the spec is not well-formed
    '''
    try:
        table_type = spec['type']
    except KeyError:
        raise InvalidDataException(f'{spec} does not have a table type')
    try:
        factory_class = cls.table_registry[table_type]['class']
    except KeyError:
      print('hello')
      raise InvalidDataException(f'{table_type} is  not have a valid table type.  Valid types are {list(cls.table_registry.keys())}')
    return factory_class.build_table(spec, *args, **kwargs)

factory_class_registry() classmethod

Return the dictionary of table types to records (class, locked). Note it returns a COPY of the registry, so it can't be accidentally overwritten Arguments: None

Source code in src/sdtp/sdtp_table_factory.py

@classmethod
def factory_class_registry(cls):
    '''
    Return the dictionary of table types to records (class, locked).  Note it returns 
    a COPY of the registry, so it can't be accidentally overwritten
    Arguments:
      None

    '''
    return cls.table_registry.copy()

get_factory(table_type) classmethod

Get the class for table_type; return None if there is no class for tabletype Arguments: table_type: the type to get the class for Returns: the class which builds table_type, or None if there is no class Raises: None

Source code in src/sdtp/sdtp_table_factory.py

@classmethod
def get_factory(cls, table_type):
    '''
    Get the class for table_type; return None if there is no class for tabletype
    Arguments:
      table_type: the type to get the class for
    Returns:
      the class which builds table_type, or None if there is no class
    Raises:
      None
    '''
    try:
        return cls.table_registry[table_type]['class']
    except KeyError:
        return None

register_factory_class(table_type, factory_class, locked=False) classmethod

Register an SDMLTableFactory class to build tables of type table_type. If locked is True, then the SDMLTableFactory class can't be overriden by a subsequent register_factory_class invocation Arguments: table_type: the type of the table factory_class: the class to build it locked (default False): if True, the factory_class can't be overwritten Returns: None Raises: InvalidDataException if the type is already registered and locked, or if factory_class is not an SDMLFactoryClass

Source code in src/sdtp/sdtp_table_factory.py

@classmethod

def register_factory_class(cls, table_type: str, factory_class, locked = False):
    '''
    Register an SDMLTableFactory class to build tables of type table_type.
    If locked is True, then the SDMLTableFactory class can't be overriden by
    a subsequent register_factory_class invocation
    Arguments:
      table_type: the type of the table
      factory_class: the class to build it
      locked (default False): if True, the factory_class can't be overwritten
    Returns:
      None
    Raises:
      InvalidDataException if the type is already registered and locked, or if factory_class
      is not an SDMLFactoryClass
    '''
    if not inspect.isclass(factory_class) or not issubclass(factory_class, SDMLTableFactory):
      raise InvalidDataException(f"{factory_class} is not a subclass of SDMLTableFactory.")

    if table_type in cls.table_registry.keys():
        record = cls.table_registry[table_type]
        if record['class'] == factory_class:
            return
        elif record['locked']:
            raise InvalidDataException(f'The factory for {table_type} is locked')
    cls.table_registry[table_type] = {'class': factory_class, 'locked': locked}

Filtering

Abstract Base

Bases: BaseModel

Abstract base class for all filters.

to_filter_spec()

Generate a dictionary form of the SDQLFilter. This is primarily for use on the client side, where A SDQLFilter can be constructed, and then a JSONified form of the dictionary version can be passed to the server for server-side filtering. It's also useful for testing and debugging Returns: A dictionary form of the Filter

Source code in src/sdtp/sdtp_filter.py

def to_filter_spec(self):
  '''
  Generate a dictionary form of the SDQLFilter.  This is primarily for use on the client side, where
  A SDQLFilter can be constructed, and then a JSONified form of the dictionary version can be passed to
  the server for server-side filtering.  It's also useful for testing and debugging
  Returns:
      A dictionary form of the Filter
  '''
  raise NotImplementedError("to_filter_spec must be implemented by subclass")

Atomic Filters

Bases: ColumnFilter

Implement an "IN_LIST" filter, which passes all rows in which the value of column is in the list given by values Arguments: values: list of values to check for

Bases: CompareFilter

Implement >=

Bases: CompareFilter

Implement >

Bases: CompareFilter

Implement <=

Bases: CompareFilter

Implement <

Bases: ColumnFilter

Implement a REGEX filter, which passes all rows in which the value of column matches the regular expression expression

Compound Filters

Bases: CompoundFilter

An ALL Filter -- matches a row if ALL of the arguments match on the column

Bases: CompoundFilter

An ANY Filter -- matches a row if ANY of the arguments match on the column

Bases: CompoundFilter

A None Filter -- matches a row if NONE of the arguments match on the column Arguments: arguments: set of subfilters

Utility Functions

Make a filter from a filter_spec. Note that filter_spec should be free of errors (run filter_spec_errors first) Arguments: filter_spec: A valid dictionary form of a filter Returns: An instance of SDQL Filters

Source code in src/sdtp/sdtp_filter.py

def make_filter(filter_spec):
  """
  Make a filter from a filter_spec.  Note that filter_spec should
  be free of errors (run filter_spec_errors first)
  Arguments:
    filter_spec: A valid dictionary form of a filter
  Returns:
    An instance of SDQL Filters
  """
  operator = filter_spec["operator"]
  if operator == "IN_RANGE":
    filter_spec = expand_in_range_spec(filter_spec)
    operator = filter_spec["operator"]

  cls = FILTER_CLASSES.get(operator)
  if not cls:
    raise ValueError(f"Unknown filter operator: {operator}")
  if issubclass(cls, CompoundFilter):
    return cls(operator = operator, arguments=[make_filter(s) for s in filter_spec["arguments"]])
  return cls(**filter_spec)

Method which checks to make sure that a filter spec is valid. Does not return, but throws an InvalidDataException with an error message if the filter spec is invalid

Parameters:

Name	Type	Description	Default
filter_spec	dict	spec to test for validity	required

Source code in src/sdtp/sdtp_filter.py

def check_valid_spec(filter_spec: dict):
  '''
  Method which checks to make sure that a filter spec is valid.
  Does not return, but throws an InvalidDataException with an error message
  if the filter spec is invalid

  Arguments:
    filter_spec (dict): spec to test for validity
  '''
  try:
    f = make_filter(filter_spec)
  except Exception as e:
    raise InvalidDataException(e)

Method which checks to make sure that a filter spec is valid. Returns True if and only if filter_spec has no errors

Parameters:

Name	Type	Description	Default
filter_spec	dict	spec to test for validity	required

Source code in src/sdtp/sdtp_filter.py

def check_valid_spec_return_boolean(filter_spec: dict):
  '''
  Method which checks to make sure that a filter spec is valid.
  Returns True if and only if filter_spec has no errors

  Arguments:
    filter_spec(dict): spec to test for validity
  '''
  try:
    f = make_filter(filter_spec)
    return True
  except Exception as e:
    return False

Filter Constants

Schema

SDML Type Constants

SDML Type Definitions

Schema Construction & Validation

Check to make sure that the Python type of val matches the implementation of sdml_type Arguments: sdml_type: an SDMLType () val: a Python value (can be anything)

Source code in src/sdtp/sdtp_schema.py

def type_check(sdml_type: str, val) -> bool:
    '''
    Check to make sure that the Python type of val matches the implementation
    of sdml_type
    Arguments:
      sdml_type: an SDMLType ()
      val:  a Python value (can be anything)
    '''
    """Check whether a value matches the given SDML type."""
    return type(val) in SDML_PYTHON_TYPES[sdml_type]

Given a list of tuples of the form (, ), return an SDTP table schema, which is a list of sdtp_schema.ColumnSpec. Raises a ValueError for an invalid type. Args: columns: List[Tuple[str, Literal["string", "number", "boolean", "date", "datetime", "timeofday"]]] Returns: The appropriate table schema Raises: ValueError if a type is not a valid sdtp_schema.SDMLType

Source code in src/sdtp/sdtp_schema.py

def make_table_schema(columns: List[Tuple[str, Literal["string", "number", "boolean", "date", "datetime", "timeofday"]]]) -> List[ColumnSpec]:
    """
    Given a list of tuples of the form (<name>, <type>), return an SDTP table schema,
    which is a list of sdtp_schema.ColumnSpec. Raises a ValueError for an invalid type.
    Args:
        columns: List[Tuple[str, Literal["string", "number", "boolean", "date", "datetime", "timeofday"]]]
    Returns:
        The appropriate table schema
    Raises:
        ValueError if a type is not a valid sdtp_schema.SDMLType
    """
    errors = [column[1] for column in columns if column[1] not in SDML_SCHEMA_TYPES]
    if len(errors) > 0:
        raise ValueError(f'Invalid types {errors} sent to make_table_schema.  Valid types are {SDML_SCHEMA_TYPES}')
    return  [{"name": column[0], "type": column[1]} for column in columns]

Returns True iff t is a valid SDML Type (["string", "number", "boolean", "date", "datetime", "timeofday"]) Argument: t: a string

Source code in src/sdtp/sdtp_schema.py

def is_valid_sdml_type(t: str) -> bool:
    '''
    Returns True iff t is a valid SDML Type (["string", "number", "boolean", "date", "datetime", "timeofday"])
    Argument:
      t: a string
    '''
    return t in SDML_SCHEMA_TYPES

Validates that the given column dictionary includes required fields and a valid SDML type. Raises ValueError if invalid. Argument: col: a dictionary

Source code in src/sdtp/sdtp_schema.py

def validate_column_spec(col: dict) -> None:
    '''
    Validates that the given column dictionary includes required fields and a valid SDML type.
    Raises ValueError if invalid.
    Argument:
      col: a dictionary
    '''
    if "name" not in col or "type" not in col:
        raise ValueError("Column spec must include 'name' and 'type'")
    if not is_valid_sdml_type(col["type"]):
        raise ValueError(f"Invalid SDML type: {col['type']}")

Validates a table schema dictionary against known SDML types and structure. Raises ValueError on failure.

Only 'schema' is allowed as the key for column definitions.

Source code in src/sdtp/sdtp_schema.py

def validate_table_schema(table_schema: dict) -> None:
    """
    Validates a table schema dictionary against known SDML types and structure.
    Raises ValueError on failure.

    Only 'schema' is allowed as the key for column definitions.
    """
    if "schema" not in table_schema:
        raise ValueError(f"Schema {table_schema} must include a 'schema' list (not 'columns')")
    if "columns" in table_schema:
        raise ValueError(
            f"Schema {table_schema} uses 'columns' — only 'schema' is supported. "
            "Please update your spec."
        )

    if not isinstance(table_schema["schema"], list):
        raise ValueError(f"{table_schema['schema']} must be a list of columns")

    for col in table_schema['schema']:
        validate_column_spec(col)

    table_type = table_schema.get("type")
    if not table_type:
        raise ValueError("Schema must include a 'type' field")

    required_fields_by_type = {
        "RemoteSDMLTable": {"url", "table_name"},
        "RowTable": {"rows"}
    }

    if table_type not in required_fields_by_type:
        raise ValueError(f"Unknown or unsupported table type: {table_type}")

    if table_type == "remote" and "auth" in table_schema:
        validate_remote_auth(table_schema["auth"])

    _check_required_fields(table_schema, table_type, required_fields_by_type[table_type])

Schema Structures

Bases: TypedDict

A column is a dictionary: {"name", "type"}

Bases: TypedDict

The base schema for a Table. A Table MUST have a type, which is a valid table, and a schema, which is a ColumnSpec list.

Bases: BaseTableSchema

The schema for a RowTable. The type of a RowTable is "RowTable", and it must have a "rows" field.

Bases: TypedDict

Specification of a Remote Authentication, for use with RemoteTables. It currently supports tokens, env variables, and files.

Bases: BaseTableSchema

The schema for a RemoteTable. The type of a RemoteTable is "RemoteSDMLTable", and it must have "url" and "table_name" fields. An auth field is optional.

Utilities

Exceptions

Bases: Exception

An exception thrown when a data table (list of rows) doesn't match an accoompanying schema, or a bad schema is specified, or a table row is the wrong length, or..

Source code in src/sdtp/sdtp_utils.py

def __init__(self, message):
    super().__init__(message)
    self.message = message

Serialization Utilities

Source code in src/sdtp/sdtp_utils.py

def json_serialize(obj):
    # Convert dates, times, and datetimes to isostrings 
    # for json serialization
    if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
        return obj.isoformat()
    raise TypeError(f"Type {type(obj)} not serializable")

Python doesn't jsonify dates, datetimes, or times properly, so convert them to isoformat strings. Return everything else as is Arguments: value -- the value to be converted column_type -- the SDTP type of the value Returns A jsonifiable form of the value

Source code in src/sdtp/sdtp_utils.py

def jsonifiable_value(value, column_type):
    '''
    Python doesn't jsonify dates, datetimes, or times properly, so
    convert them to isoformat strings.  Return everything else as is
    Arguments:
        value -- the value to be converted
        column_type -- the SDTP type of the value
    Returns
        A jsonifiable form of the value
    '''
    if column_type in NON_JSONIFIABLE_TYPES:
        return value.isoformat()
    else:
        return value

IReturn the jsonified form of the row, using jsonifiable_value for each element Arguments: row -- the row to be converted column_types -- the types of each element of the row Returns A row of jsonifiable values

Source code in src/sdtp/sdtp_utils.py

def jsonifiable_row(row, column_types):
    '''
    IReturn the jsonified form of the row, using jsonifiable_value for each element
    Arguments:
        row -- the row to be converted
        column_types -- the types of each element of the row
    Returns
        A row of jsonifiable values
    '''
    return [jsonifiable_value(row[i], column_types[i]) for i in range(len(row))]

Return the jsonifiable form of the list of rows, using jasonifiable_row for each row Arguments: rows -- the list of rows to be converted column_types -- the types of each element of the row Returns A list of rows of jsonified values

Source code in src/sdtp/sdtp_utils.py

def jsonifiable_rows(rows, column_types):
    '''
    Return the jsonifiable form of the list of rows, using jasonifiable_row for each row
    Arguments:
        rows -- the list of rows to be converted
        column_types -- the types of each element of the row
    Returns
        A list of rows  of jsonified values
    '''
    return [jsonifiable_row(row, column_types) for row in rows]

Return a jsonifiable version of the column of values, using jsonifiable_value to do the conversion. We actually cheat a little, only calling _jsonifiable_value if column_type is one of SDML_TIME, "date", "datetime"

Source code in src/sdtp/sdtp_utils.py

def jsonifiable_column(column, column_type):
    '''
    Return a jsonifiable version of the column of values, using jsonifiable_value
    to do the conversion.  We actually cheat a little, only calling _jsonifiable_value if column_type
    is one of SDML_TIME, "date", "datetime"
    '''
    if column_type in NON_JSONIFIABLE_TYPES:
        return [jsonifiable_value(value, column_type) for value in column]
    else:
        return column

Type Checking & Conversion

Check to make sure the values in list_of_values are all the right Python type for operations. Arguments: sdml_type: One of SDML_SCHEMA_TYPES list_of_values: a Python list to be tested

Source code in src/sdtp/sdtp_utils.py

def check_sdml_type_of_list(sdml_type, list_of_values):
    '''
    Check to make sure the values in list_of_values are all the right Python 
    type for operations.
    Arguments:
        sdml_type: One of SDML_SCHEMA_TYPES
        list_of_values: a Python list to be tested
    '''
    type_check_list = [type_check(sdml_type, val) for val in list_of_values]
    return not (False in type_check_list)

Convert value to sdml_type, so that comparisons can be done. This is used to convert the values in a filter_spec to a form that can be used in a filter. Throws an InvalidDataException if the type can't be converted. An exception is Boolean, where "True, true, t" are all converted to True, but any other values are converted to False

Parameters:

Name	Type	Description	Default
sdml_type	str	type to convert to	required
value	Any	value to be converted	required

Returns: value cast to the correct type

Source code in src/sdtp/sdtp_utils.py

def convert_to_type(sdml_type, value):
    '''
    Convert value to sdml_type, so that comparisons can be done.  This is used to convert
    the values in a filter_spec to a form that can be used in a filter.
    Throws an InvalidDataException if the type can't be converted.
    An exception is Boolean, where "True, true, t" are all converted to True, but any
    other values are converted to False

    Arguments:
        sdml_type (str): type to convert to
        value (Any): value to be converted
    Returns:
        value cast to the correct type
    '''
    if type(value) in SDML_PYTHON_TYPES[sdml_type]:
        return value
    if sdml_type == "string":
        if isinstance(value, str):
            return value
        try:
            return str(value)
        except ValueError:
            raise InvalidDataException('Cannot convert value to string')
    elif sdml_type == "number":
        return _convert_to_number(value)
    elif sdml_type == "boolean":
        if isinstance(value, bool):
            return value
        if isinstance(value, str):
            return value in {'True', 'true', 't', '1', '1.0'}
        if isinstance(value, int):
            return value != 0
        if isinstance(value, float):
            return value != 0.0
        return False
    # Everything else is a date or time

    elif sdml_type == "datetime":
        if type(value) == datetime.date:
            return datetime.datetime(value.year, value.month, value.day, 0, 0, 0)
        if isinstance(value, str):
            try:
                return datetime.datetime.fromisoformat(value)
            except Exception:
                raise InvalidDataException(f"Can't convert {value} to datetime")
        raise InvalidDataException(f"Can't convert {value} to datetime")

    elif sdml_type == "date":
        if type(value) in SDML_PYTHON_TYPES["datetime"]:
            return value.date()
        if isinstance(value, str):
            try:
                return datetime.datetime.fromisoformat(value).date()
            except Exception:
                raise InvalidDataException(f"Can't convert {value} to date")
        raise InvalidDataException(f"Can't convert {value} to date")

    elif sdml_type == "timeofday":
        if type(value) in SDML_PYTHON_TYPES["datetime"]:
            return value.time()
        if isinstance(value, str):
            try:
                return datetime.time.fromisoformat(value)
            except Exception:
                try:
                    return datetime.datetime.fromisoformat(value).time()
                except Exception:
                    raise InvalidDataException(f"Can't convert {value} to time")

        raise InvalidDataException(f"Couldn't convert {value} to {sdml_type}")

    raise InvalidDataException(f"Don't recognize {sdml_type}")

Convert value_list to sdml_type, so that comparisons can be done. Currently only works for lists of string, number, and boolean. Returns a default value if value can't be converted Note that it's the responsibility of the object which provides the rows to always provide the correct types, so this really should always just return a new copy of value_list Arguments: sdml_type: type to convert to value_list: list of values to be converted Returns: value_list with each element cast to the correct type

Source code in src/sdtp/sdtp_utils.py

def convert_list_to_type(sdml_type, value_list):
    '''
    Convert value_list to sdml_type, so that comparisons can be done.  Currently only works for lists of string, number, and boolean.
    Returns a default value if value can't be converted
    Note that it's the responsibility of the object which provides the rows to always provide the correct types,
    so this really should always just return a new copy of value_list
    Arguments:
        sdml_type: type to convert to
        value_list: list of values to be converted
    Returns:
        value_list with each element cast to the correct type
    '''
    try:
        return  [convert_to_type(sdml_type, elem) for elem in value_list]

        # result = []
        # for i in range(len(value_list)): result.append(convert_to_type(sdml_type, value_list[i]))
        # return result
    except Exception as exc:
        raise InvalidDataException(f'Failed to convert {value_list} to {sdml_type}')

Source code in src/sdtp/sdtp_utils.py

def convert_row_to_type_list(sdml_type_list, row):
    # called from convert_rows_to_type_list, which should error check
    # to make sure that the row is the same length as sdml_type_list
    return [convert_to_type(sdml_type_list[i], row[i]) for i in range(len(row))]

Convert the list of rows to the

Source code in src/sdtp/sdtp_utils.py

def convert_rows_to_type_list(sdml_type_list, rows):
    '''
    Convert the list of rows to the 
    '''
    length = len(sdml_type_list)


    for row in rows:
        # print("convert_rows_to_type_list received types:", sdml_type_list)
        # print("Row:", row)
        # print("Expected length:", length, "Actual length:", len(row))

        if len(row) != length:
            raise InvalidDataException(f'Length mismatch: required number of columns {length}, length {row} = {len(row)}')
    return  [convert_row_to_type_list(sdml_type_list, row) for row in rows]

Convert value_dict to sdml_type, so that comparisons can be done. Currently only works for lists of string, number, and boolean.

Returns a default value if value can't be converted Note that it's the responsibility of the object which provides the rows to always provide the correct types, so this really should always just return a new copy of value_list Arguments: sdml_type: type to convert to value_dict: dictionary of values to be converted Returns: value_dict with each value in the dictionary cast to the correct type

Source code in src/sdtp/sdtp_utils.py

def convert_dict_to_type(sdml_type, value_dict):
    '''
    Convert value_dict to sdml_type, so that comparisons can be done.  Currently only works for lists of string, number, and boolean.

    Returns a default value if value can't be converted
    Note that it's the responsibility of the object which provides the rows to always provide the correct types,
    so this really should always just return a new copy of value_list
    Arguments:
        sdml_type: type to convert to
        value_dict: dictionary of values to be converted
    Returns:
        value_dict with each value in the dictionary cast to the correct type
    '''
    result = {}
    try:
        for (key, value) in value_dict.items():
            result[key] = convert_to_type(sdml_type, value)
        return result
    except Exception as exc:
        raise InvalidDataException(f'Failed to convert {value_dict} to {sdml_type}')

Authentication Utilities

Bases: TypedDict

The authentication token is in an the environment variable env

Bases: TypedDict

The authentication token is in the file at path

Bases: TypedDict

The authentication token is the value

Resolve an AuthMethod dict to a credential string. Returns None if the method can't be satisfied (env var/file missing, etc.).

Source code in src/sdtp/sdtp_utils.py

def resolve_auth_method(method: AuthMethod) -> Optional[str]:
    """
    Resolve an AuthMethod dict to a credential string.
    Returns None if the method can't be satisfied (env var/file missing, etc.).
    """
    if "env" in method:
        return os.environ.get(method["env"])
    elif "path" in method:
        try:
            with open(os.path.expanduser(method["path"]), "r") as f:
                return f.read().strip()
        except Exception:
            return None
    elif "value" in method:
        return method["value"]
    return None

Table Server

Exceptions

Bases: Exception

An exception that is thrown when a table is not found in the TableServer

Source code in src/sdtp/table_server.py

def __init__(self, message):
    super().__init__(message)

Bases: Exception

An exception that is thrown when a column is not found for a specific table

Source code in src/sdtp/table_server.py

def __init__(self, message):
    super().__init__(message)

Table Server

The server for tables. Its task is to maintain a correspondence between table names and the actual tables. It also maintains the security information for a table (the variables and values required to access the table), and gives column information across tables

Source code in src/sdtp/table_server.py

def __init__(self):
    self.servers = {}
    self.factories = {}
    self.loaders = {
        "file": FileTableLoader,
        "uri": HTTPTableLoader,
        # Add additional loaders here as needed
    }

add_sdtp_table(table_name, sdtp_table)

Register a SDMLTable to serve data for a specific table name. Raises an InvalidDataException if table_name is None or sdtp_table is None or is not an instance of SDMLTable.

Parameters:

Name	Type	Description	Default
table_name	str	name of the table	required
sdtp_table	SDMLTable	table to add	required

Source code in src/sdtp/table_server.py

def add_sdtp_table(self, table_name, sdtp_table):
    '''
    Register a SDMLTable to serve data for a specific table name.
    Raises an InvalidDataException if table_name is None or sdtp_table is None or is not an instance of SDMLTable.

    Arguments:
        table_name (str): name of the table
        sdtp_table (SDMLTable): table to add

    '''
    _check_type(sdtp_table, SDMLTable, 'The sdtp_table argument to add_sdtp_table must be a Table, not')
    self.servers[table_name] = sdtp_table

add_sdtp_table_from_dictionary(name, table_dictionary)

Add an SDMLTable from a dictionary (intermediate on-disk form). The table dictionary has fields schema and type, and then type- specific fields. Calls TableBuilder to build the table, then calls self.add_sdtp_table to add the table. Raises an InvalidDataException if self.add_sdtp_table or TableBuilder.buildTable raises it Arguments: name (str): the name of the table table_dictionary (dict): dictionary of the form {"type", "table"}, where table is a table specification: a dictionary with the fields type and schema

Source code in src/sdtp/table_server.py

def add_sdtp_table_from_dictionary(self, name, table_dictionary):
    '''
    Add an  SDMLTable from a dictionary (intermediate on-disk form).   The table dictionary has fields schema and type, and then type-
    specific fields.  Calls TableBuilder to build the table,
    then calls self.add_sdtp_table to add the table.
    Raises an InvalidDataException if self.add_sdtp_table or TableBuilder.buildTable raises it 
    Arguments:
        name (str): the name of the table
        table_dictionary (dict): dictionary of the form {"type", "table"}, where table is a table specification: a dictionary
                         with the fields type and schema

    '''


    table = TableBuilder.build_table(table_dictionary)
    self.add_sdtp_table(name,  table)

get_all_tables()

Get all the tables. This is to support a request for a numeric_spec or all_values for a column name when the table_name is not specified. In this case, all tables will be searched for this column name.

Returns:

Name	Type	Description
list	list	a list of all tables

Source code in src/sdtp/table_server.py

def get_all_tables(self) -> list:
    '''
    Get all the tables.  This
    is to support a request for a numeric_spec or all_values for a column name when the
    table_name is not specified. In this case, all tables will be searched for this column name.


    Returns:
        list: a list of all tables
    '''
    tables = self.servers.values()
    return list(tables)

get_all_values(table_name, column_name)

Get all of the distinct values for column column_name for table table_name. Returns the list of distinct values for the columns Arguments: table_name (str): table to be searched column_name (str): name of the column

Returns:

Name	Type	Description
list	list	Returns the list of distinct values for the columns

Raises: TableNotFoundException if the table is not found ColumnNotFoundException if the column can't be found

Source code in src/sdtp/table_server.py

def get_all_values(self, table_name, column_name) -> list:
    '''
    Get all of the distinct values for column column_name for table
    table_name.  Returns the list of distinct values for the columns
    Arguments:
        table_name (str): table to be searched
        column_name (str): name of the column

    Returns:
        list: Returns the list of distinct values for the columns
    Raises:
        TableNotFoundException if the table is not found
        ColumnNotFoundException if the column can't be found
    '''

    _check_type(column_name, str, 'Column name must be a string, not')
    table = self.get_table(table_name)  # Note this will throw the TableNotFoundException

    try:
        return table.all_values(column_name)
    except InvalidDataException:
        raise ColumnNotFoundException(f'Column {column_name} not found in table {table_name}')

get_column(table_name, column_name)

Get the column for column column_name for table table_name. Returns the column as a list Arguments: table_name: table to be searched column_name: name of the column

Returns:

Name	Type	Description
list	list	Returns a dictionary with keys{max_val, min_val}

Raises: TableNotFoundException if the table is not found ColumnNotFoundException if the column can't be found

Source code in src/sdtp/table_server.py

def get_column(self, table_name, column_name) -> list:
    '''
    Get the column for column column_name for table
    table_name.  Returns the column as a list
    Arguments:
        table_name: table to be searched
        column_name: name of the column

    Returns:
        list: Returns  a dictionary with keys{max_val, min_val}
    Raises:
        TableNotFoundException if the table is not found
        ColumnNotFoundException if the column can't be found
    '''
    _check_type(column_name, str, 'Column name must be a string, not')
    table = self.get_table(table_name)  # Note this will throw the TableNotFoundException
    try:
        return table.get_column(column_name)
    except InvalidDataException:
        raise ColumnNotFoundException(f'Column {column_name} not found in table {table_name}')

get_range_spec(table_name, column_name)

Get the range specification for column column_name for table table_name. Returns a two-length list [min_val, max_val] Arguments: table_name: table to be searched column_name: name of the column

Returns:

Name	Type	Description
list	list	Returns a dictionary with keys{max_val, min_val}

Raises: TableNotFoundException if the table is not found ColumnNotFoundException if the column can't be found

Source code in src/sdtp/table_server.py

def get_range_spec(self, table_name, column_name) -> list:
    '''
    Get the range specification for column column_name for table
    table_name.  Returns  a two-length list [min_val, max_val]
    Arguments:
        table_name: table to be searched
        column_name: name of the column

    Returns:
        list: Returns  a dictionary with keys{max_val, min_val}
    Raises:
        TableNotFoundException if the table is not found
        ColumnNotFoundException if the column can't be found
    '''
    _check_type(column_name, str, 'Column name must be a string, not')
    table = self.get_table(table_name)  # Note this will throw the TableNotFoundException
    try:
        return table.range_spec(column_name)
    except InvalidDataException:
        raise ColumnNotFoundException(f'Column {column_name} not found in table {table_name}')

get_table(table_name)

Get the table with name table_name, first checking to see if table access is authorized by the passed headers. Arguments: table_name: name of the table to search for

Returns:

Name	Type	Description
SDMLTable	SDMLTable	The SDML table corresponding to the request

Raises: TableNotFoundException if the table is not found

Source code in src/sdtp/table_server.py

def get_table(self, table_name) -> SDMLTable:
    '''
    Get the table with name table_name, first checking to see
    if  table access is authorized by the passed headers.
    Arguments:
        table_name: name of the table to search for

    Returns:
        SDMLTable: The SDML table corresponding to the request
    Raises:
        TableNotFoundException if the table is not found

    '''
    try:
        return self.servers[table_name]

    except KeyError:
        raise TableNotFoundException(f'Table {table_name} not found')

init_from_config(config_path)

Initialize TableServer from config file. Config must be a JSON list as specified above.

Source code in src/sdtp/table_server.py

def init_from_config(self, config_path):
    """
    Initialize TableServer from config file.
    Config must be a JSON list as specified above.
    """
    with open(config_path, "r") as f:
        config = load(f)

    for entry in config:
        name = entry["name"]
        load_spec = entry["load_spec"]

        location_type = load_spec["location_type"]
        loader_cls = self.loaders.get(location_type)
        if loader_cls is None:
            raise ValueError(f"No loader for location_type '{location_type}'")

        loader = loader_cls(load_spec)
        table_spec = loader.load()  # Should return a dict

        # Figure out the table type (e.g. "row", "remote") from the spec
        table = TableBuilder.build_table(table_spec)
        self.add_sdtp_table(name, table)

Table Loader Adapters

Bases: ABC

load() abstractmethod

Returns a dict spec for the table

Source code in src/sdtp/table_server.py

@abstractmethod
def load(self):
    """Returns a dict spec for the table"""

Bases: TableLoader

Loads a table from a path

Source code in src/sdtp/table_server.py

def __init__(self, spec):
    self.path = spec["path"]

Bases: TableLoader

Source code in src/sdtp/table_server.py

def __init__(self, spec):
    self.url = spec["url"]
    self.auth = HeaderInfo(spec["auth_info"]) if "auth_info" in spec else None

Supports loading headers from file or env according to our spec: - {"from_file": "path/to/headers.json"} - {"headers": {"Authorization": {"from_env": "API_AUTH"}}}

Source code in src/sdtp/table_server.py

def __init__(self, spec, error_on_missing_env=True):
    self.headers_dict = {}

    if spec is None:
        return

    # Case 1: Load headers from a JSON file
    if "from_file" in spec:
        if not os.path.exists(spec["from_file"]):
            raise FileNotFoundError(f"Header file not found: {spec['from_file']}")
        with open(spec["from_file"], "r") as f:
            data = load(f)
            if not isinstance(data, dict):
                raise ValueError(f"Headers file must be a JSON object (dict), got {type(data)}")
            # No inline validation: assume secret file is trusted
            self.headers_dict = data

    # Case 2: Load each header from an env var
    elif "headers" in spec:
        headers_spec = spec["headers"]
        if not isinstance(headers_spec, dict):
            raise ValueError(f"'headers' must be a dict, got {type(headers_spec)}")
        for k, v in headers_spec.items():
            if isinstance(v, dict) and "from_env" in v:
                env_key = v["from_env"]
                env_val = os.environ.get(env_key)
                if env_val is None:
                    msg = f"Environment variable '{env_key}' for header '{k}' is not set."
                    if error_on_missing_env:
                        raise EnvironmentError(msg)
                    else:
                        print(f"Warning: {msg} (header omitted)")
                        continue
                self.headers_dict[k] = env_val
            else:
                raise ValueError(f"Header '{k}' value must be a dict with 'from_env', got {v}")
    else:
        raise ValueError("auth_info must contain 'from_file' or 'headers' as its only key.")

    # Strict: Disallow extra keys
    allowed_keys = {"from_file", "headers"}
    extra_keys = set(spec.keys()) - allowed_keys
    if extra_keys:
        raise ValueError(f"auth_info has unsupported keys: {extra_keys}")

SDTP Web Server

The SDTPServer blueprint exposes the following REST API endpoints:

• /get_table_names
• /get_tables
• /get_table_schema
• /get_filtered_rows
• /get_range_spec
• /get_all_values
• /get_column

All endpoints return JSON; see the SDTP protocol docs for usage.

Flask Blueprint

Bases: Blueprint

An SDTP Server. This is just an overlay on a Flask Blueprint, added so we can expose initialize methods to the application

Source code in src/sdtp/sdtp_server.py

def __init__(self, name, __name__):
    super(SDTPServer, self).__init__(name, __name__)
    self.table_server = TableServer()
    self.logger = None
    self.ROUTES = [
        {"url": "/get_table_names", "method": "GET", "headers": "None",
            "description": "Return the list of table names"},
        {"url": "/get_table_schema?table_name string, required", "method": "GET", "headers": "None",
            "description": 'Returns the schema of the table as a list of objects.  Each object will  contain the fields "name" and "type", where "type"is an SDML type.'},
        {"url": "/get_tables", "method": "GET", "headers": "None",
            "description": 'Dumps a JSONIfied dictionary of the form:{table_name: <table_schema>}, where <table_schema> is a dictionary{"name": name, "type": type}'},
        {"url": "/get_filtered_rows", "method": "POST",
            "body": {"table": " required, the name of the table to get the rows from",
                    "columns": " If  present, a list of the names of the columns to fetch",
                    "filter": " optional, a filter_spec in the SDTP filter language"},

            "description": "Get the rows from table table which match filter filter.  If columns is present, return only those columns.  Returns a simple list of lists of columns"},
        {"url": "/get_range_spec?column_name string, required&table_name string, required", "method": "GET",
            "headers": "None",
            "description": "Get the  minimum and maximum values for column column_name in table table_name, returned as a list [min_val, max_val]."},
        {"url": "/get_all_values?column_name string, required&table_name string, required", "method": "GET",
            "headers": "None",
            "description": "Get all the distinct values for column column_name in table table_name, returned as a sorted list."},
        {"url": "/get_column?column_name string, required&table_name string, required", "method": "GET",
            "headers": "None",
            "description": "Return the column <column_name> in table <table_name> as a list.  The behavior is undefined when the table is infinite"},

    ]

SDTP Client

Exceptions

Bases: Exception

Custom exception for SDTP client errors.

Configuration

Loads and validates the SDTPClient YAML config file.

Returns:

Type	Description
Dict[str, Any]	Dict[str, AuthMethod] -- The credentials mapping (URL -> AuthMethod).

Raises: Exception on parse or validation error.

Source code in src/sdtp/sdtp_client.py

def load_config(config_path: Optional[str] = None) -> Dict[str, Any]:
    """
    Loads and validates the SDTPClient YAML config file.

    Returns:
        Dict[str, AuthMethod] -- The credentials mapping (URL -> AuthMethod).
    Raises:
        Exception on parse or validation error.
    """
    if config_path is None:
        config_path = os.environ.get("SDTP_CLIENT_CONFIG", DEFAULT_CONFIG_PATH)
    config_path = os.path.expanduser(config_path)
    if not os.path.exists(config_path):
        # raise FileNotFoundError(f"Config file not found: {config_path}")
        print(f"Warning: No SDTP config file found at {config_path}. Proceeding with no credentials.")
        return {} # Or None

    with open(config_path, "r") as f:
        config = yaml.safe_load(f)

    # Expect config with a 'credentials' mapping
    creds = config.get("credentials")
    if not isinstance(creds, dict):
        raise ValueError("Missing or invalid 'credentials' section in config.")

    for url, method in creds.items():
        if not isinstance(method, dict) or len(method) != 1 or not any(
            k in method for k in ("env", "path", "value")
        ):
            raise ValueError(f"Invalid AuthMethod for {url}: {method}")
    return creds  # type: Dict[str, AuthMethod]

Main Client

Minimal SDTP Client: Connects to SDTP REST endpoints with robust, flexible authentication.

Credential Discovery & Authentication

SDTPClient uses a YAML config file as a minimal password manager, mapping each SDTP server URL to a credential retrieval method:

• env: Get the credential from a named environment variable (supports ephemeral tokens from IdP or orchestration).
• path: Read the credential from a file (works with secrets managers or file mounts).
• value: Use the explicit token value (for dev/test only).

The config file is read at instantiation (default: ~/.sdtp_client_config.yaml), and may be overridden by the SDTP_CLIENT_CONFIG env var or config_path argument.

For each API call

1. If an explicit auth argument is supplied, it overrides all other methods for that call only.
2. Otherwise, the credential method for the client's server URL is looked up in the config file.
3. If no entry is found for the URL, the 'default' entry (if any) is used.
4. If no credential method is found, no Authorization header is sent (anonymous access).

All credential methods are re-evaluated at each call — changes to env vars or files are picked up automatically.

Power users may also specify headers per method to override or add HTTP headers directly.

Convenience Methods

SDTPClient provides high-level helpers to build table schemas, filter specs, and DataFrames, minimizing boilerplate and making SDTP more accessible for data science and ETL workflows.

See the user documentation for full config format, method details, and best practices.

Parameters:

Name	Type	Description	Default
url	str	Base URL of the SDTP server (e.g., "http://localhost:5000")	required
config_path	Optional[str]	Optional path to the YAML credential config file (default: "~/.sdtp_client_config.yaml").	'~/.sdtp_client_config.yaml'
auth	Optional[AuthMethod]	Optional explicit AuthMethod for this client (overrides config).	None

Source code in src/sdtp/sdtp_client.py

def __init__(
    self,
    url: str,
    config_path: Optional[str] = "~/.sdtp_client_config.yaml",
    auth: Optional[AuthMethod] = None
):
    """
    Args:
        url: Base URL of the SDTP server (e.g., "http://localhost:5000")
        config_path: Optional path to the YAML credential config file (default: "~/.sdtp_client_config.yaml").
        auth: Optional explicit AuthMethod for this client (overrides config).
    """
    self.url = url.rstrip("/")
    self.credentials = load_config(config_path)
    self.auth = None
    if auth:
        self.auth = auth
    else:
        self.auth = _query_credential_method(self.credentials, self.url)

clear()

Clear the stored credential method for this client. After calling, further calls will use config or explicit per-method auth.

Source code in src/sdtp/sdtp_client.py

def clear(self):
  """
  Clear the stored credential method for this client.
  After calling, further calls will use config or explicit per-method auth.
  """
  self.auth = None

echo_json_post(payload, auth=None)

POST /_echo_json_post — Echoes posted JSON (debug/testing only).

Parameters:

Name	Type	Description	Default
payload	dict	Payload in JSON form.	required
auth	Optional[Union[str, AuthMethod]]	Auth token or method for this request (overrides all others).	None

Returns:

Type	Description
dict	The echoed payload (dict).

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def echo_json_post(
    self,
    payload: dict,
    auth: Optional[Union[str, AuthMethod]] = None,
) -> dict:
    """
    POST /_echo_json_post — Echoes posted JSON (debug/testing only).

    Args:
        payload: Payload in JSON form.
        auth: Auth token or method for this request (overrides all others).

    Returns:
        The echoed payload (dict).

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    headers = {'Accept': 'application/json'}
    token = self._resolve_effective_token(auth)
    if token:
        headers['Authorization'] = f'Bearer {token}'
    response = requests.post(
        f"{self.url}/_echo_json_post",
        json=payload,
        headers=headers
    )
    return _process_response(response)

get_all_values(table_name, column_name, auth=None)

Returns all distinct values for a column.

Parameters:

Name	Type	Description	Default
table_name	str	Table containing the column.	required
column_name	str	Column to get values for.	required
auth	Optional[Union[str, AuthMethod]]	Optional string token or AuthMethod dict for this call.	None

Returns:

Type	Description
List[Any]	List[Any]: List of column values.

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def get_all_values(
    self,
    table_name: str,
    column_name: str,
    auth: Optional[Union[str, AuthMethod]] = None
) -> List[Any]:
    """
    Returns all distinct values for a column.

    Args:
        table_name: Table containing the column.
        column_name: Column to get values for.
        auth: Optional string token or AuthMethod dict for this call.

    Returns:
        List[Any]: List of column values.

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    return self._execute_column_query('get_all_values', table_name, column_name, auth)

get_column(table_name, column_name, auth=None)

Returns the entire column as a list.

Parameters:

Name	Type	Description	Default
table_name	str	Table containing the column.	required
column_name	str	Column to get.	required
auth	Optional[Union[str, AuthMethod]]	Optional string token or AuthMethod dict for this call.	None

Returns:

Type	Description
List[Any]	List[Any]: The column as a list.

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def get_column(
    self,
    table_name: str,
    column_name: str,
    auth: Optional[Union[str, AuthMethod]] = None
) -> List[Any]:
    """
    Returns the entire column as a list.

    Args:
        table_name: Table containing the column.
        column_name: Column to get.
        auth: Optional string token or AuthMethod dict for this call.

    Returns:
        List[Any]: The column as a list.

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    return self._execute_column_query('get_column', table_name, column_name, auth)

get_filtered_rows(table, columns=None, filter=None, result_format=None, auth=None)

POST /get_filtered_rows — Fetch filtered rows from a table.

Parameters:

Name	Type	Description	Default
table	str	Name of the table (required).	required
columns	Optional[List[str]]	List of columns to return (optional).	None
filter	Optional[Dict[str, Any]]	SDTP filter spec (optional).	None
result_format	Optional[str]	Output format (optional).	None
auth	Optional[Union[str, AuthMethod]]	Auth token or method for this request (overrides all others).	None

Returns:

Type	Description
Any	List of rows (list of lists), or as specified by result_format.

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def get_filtered_rows(
    self,
    table: str,
    columns: Optional[List[str]] = None,
    filter: Optional[Dict[str, Any]] = None,
    result_format: Optional[str] = None,
    auth: Optional[Union[str, AuthMethod]] = None,
) -> Any:
    """
    POST /get_filtered_rows — Fetch filtered rows from a table.

    Args:
        table: Name of the table (required).
        columns: List of columns to return (optional).
        filter: SDTP filter spec (optional).
        result_format: Output format (optional).
        auth: Auth token or method for this request (overrides all others).

    Returns:
        List of rows (list of lists), or as specified by result_format.

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    headers = {'Accept': 'application/json'}
    token = self._resolve_effective_token(auth)
    if token:
        headers['Authorization'] = f'Bearer {token}'
    body: Dict[str, Any] = {"table": table}
    if columns is not None:
        body["columns"] = columns
    if filter is not None:
        body["filter"] = filter
    if result_format is not None:
        body["result_format"] = result_format

    response = requests.post(
        f"{self.url}/get_filtered_rows",
        json=body,
        headers=headers
    )
    return _process_response(response)

get_range_spec(table_name, column_name, auth=None)

Returns [min, max] for a column.

Parameters:

Name	Type	Description	Default
table_name	str	Table containing the column.	required
column_name	str	Column to get min/max for.	required
auth	Optional[Union[str, AuthMethod]]	Optional string token or AuthMethod dict for this call.	None

Returns:

Type	Description
List[Any]	List[Any]: List of length 2: [min, max]

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def get_range_spec(
    self,
    table_name: str,
    column_name: str,
    auth: Optional[Union[str, AuthMethod]] = None
) -> List[Any]:
    """
    Returns [min, max] for a column.

    Args:
        table_name: Table containing the column.
        column_name: Column to get min/max for.
        auth: Optional string token or AuthMethod dict for this call.

    Returns:
        List[Any]: List of length 2: [min, max]

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    return self._execute_column_query('get_range_spec', table_name, column_name, auth)

get_table_schema(table_name, auth=None)

Returns the schema for a table.

Parameters:

Name	Type	Description	Default
table_name	str	Table to get the schema for.	required
auth	Optional[Union[str, AuthMethod]]	Optional string token or AuthMethod dict for this call.	None

Returns:

Type	Description
List[Dict[str, str]]	List[Dict[str, str]]: Schema for the table.

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def get_table_schema(
    self,
    table_name: str,
    auth: Optional[Union[str, AuthMethod]] = None
) -> List[Dict[str, str]]:
    """
    Returns the schema for a table.

    Args:
        table_name: Table to get the schema for.
        auth: Optional string token or AuthMethod dict for this call.

    Returns:
        List[Dict[str, str]]: Schema for the table.

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    query = f"{self.url}/get_table_schema?table_name={table_name}"
    return self._do_get_request(query, auth)

get_tables(auth=None)

Returns a dictionary indexed by table name with values the schema for each table.

Parameters:

Name	Type	Description	Default
auth	Optional[Union[str, AuthMethod]]	Optional string token or AuthMethod dict to use for this call.	None

Returns:

Type	Description
Dict[str, Dict]	Dict[Dict]: Dictionary of table schemas

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def get_tables(self, auth: Optional[Union[str, AuthMethod]] = None) -> Dict[str, Dict]:
    """
    Returns a dictionary indexed by table name with values the schema for each table.

    Args:
        auth: Optional string token or AuthMethod dict to use for this call.

    Returns:
        Dict[Dict]:  Dictionary of table schemas

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    query = f'{self.url}/get_tables'
    return self._do_get_request(query, auth)

list_tables(auth=None)

Returns a list of table names from the server.

Parameters:

Name	Type	Description	Default
auth	Optional[Union[str, AuthMethod]]	Optional string token or AuthMethod dict to use for this call.	None

Returns:

Type	Description
List[str]	List[str]: list of table names.

Raises:

Type	Description
SDTPClientError	SDTPClientError on a bad response.

Source code in src/sdtp/sdtp_client.py

def list_tables(self, auth: Optional[Union[str, AuthMethod]] = None) -> List[str]:
    """
    Returns a list of table names from the server.

    Args:
        auth: Optional string token or AuthMethod dict to use for this call.

    Returns:
        List[str]: list of table names.

    Raises:
        SDTPClientError: SDTPClientError on a bad response.
    """
    query = f'{self.url}/get_table_names'
    return self._do_get_request(query, auth)

query_credential_method(url=None)

Return the authentication method (AuthMethod) for the specified URL, as stored in this client's credentials mapping.

Parameters:

Name	Type	Description	Default
url	Optional[str]	The server URL to query (defaults to this client's URL if None).	None

Returns:

Type	Description
Optional[AuthMethod]	The AuthMethod dict (e.g., {'env': ...}, {'path': ...}, or {'value': ...}),
Optional[AuthMethod]	or None if no method is configured for this URL.

Example

method = client.query_credential_method()

returns this client's method

method = client.query_credential_method("https://other.example.com")

returns method for another URL (if in credentials)

Source code in src/sdtp/sdtp_client.py

def query_credential_method(self, url: Optional[str] = None) -> Optional[AuthMethod]:
    """
    Return the authentication method (AuthMethod) for the specified URL, as stored in this client's credentials mapping.

    Args:
        url: The server URL to query (defaults to this client's URL if None).

    Returns:
        The AuthMethod dict (e.g., {'env': ...}, {'path': ...}, or {'value': ...}),
        or None if no method is configured for this URL.

    Example:
        method = client.query_credential_method()
        # returns this client's method

        method = client.query_credential_method("https://other.example.com")
        # returns method for another URL (if in credentials)
    """
    effective_url = url if url else self.url
    return _query_credential_method(self.credentials, effective_url)

table_exists(table_name, auth=None)

Return True if and only if table table_name exists on the server Args: table_name: Table to check. auth: Optional string token or AuthMethod dict for this call. Returns: bool: True if the table exists, False otherwise.

Source code in src/sdtp/sdtp_client.py

def table_exists(self, table_name: str, auth: Optional[Union[str, AuthMethod]] = None)->bool:
    """
    Return True if and only if table table_name exists on the server
    Args:
        table_name: Table to check.
        auth: Optional string token or AuthMethod dict for this call.
    Returns:
        bool: True if the table exists, False otherwise.
    """
    names = self.get_tables(auth).keys()
    return table_name in names