diff --git a/apps/beeswax/thrift/include/hive_metastore.thrift b/apps/beeswax/thrift/include/hive_metastore.thrift index 4e9d42e..c8dedb9 100755 --- a/apps/beeswax/thrift/include/hive_metastore.thrift +++ b/apps/beeswax/thrift/include/hive_metastore.thrift @@ -44,10 +44,66 @@ struct Type { 4: optional list fields // if the name is one of the user defined types } +enum HiveObjectType { + GLOBAL = 1, + DATABASE = 2, + TABLE = 3, + PARTITION = 4, + COLUMN = 5, +} + +enum PrincipalType { + USER = 1, + ROLE = 2, + GROUP = 3, +} + +struct HiveObjectRef{ + 1: HiveObjectType objectType, + 2: string dbName, + 3: string objectName, + 4: list partValues, + 5: string columnName, +} + +struct PrivilegeGrantInfo { + 1: string privilege, + 2: i32 createTime, + 3: string grantor, + 4: PrincipalType grantorType, + 5: bool grantOption, +} + +struct HiveObjectPrivilege { + 1: HiveObjectRef hiveObject, + 2: string principalName, + 3: PrincipalType principalType, + 4: PrivilegeGrantInfo grantInfo, +} + +struct PrivilegeBag { + 1: list privileges, +} + +struct PrincipalPrivilegeSet { + 1: map> userPrivileges, // user name -> privilege grant info + 2: map> groupPrivileges, // group name -> privilege grant info + 3: map> rolePrivileges, //role name -> privilege grant info +} + +struct Role { + 1: string roleName, + 2: i32 createTime, + 3: string ownerName, +} + // namespace for tables struct Database { 1: string name, 2: string description, + 3: string locationUri, + 4: map parameters, // properties associated with the database + 5: optional PrincipalPrivilegeSet privileges } // This object holds the information needed by SerDes @@ -87,7 +143,11 @@ struct Table { 6: i32 retention, // retention time 7: StorageDescriptor sd, // storage descriptor of the table 8: list partitionKeys, // partition keys of the table. only primitive types are supported - 9: map parameters // to store comments or any other user level parameters + 9: map parameters, // to store comments or any other user level parameters + 10: string viewOriginalText, // original view text, null for non-view + 11: string viewExpandedText, // expanded view text, null for non-view + 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE + 13: optional PrincipalPrivilegeSet privileges, } struct Partition { @@ -97,18 +157,21 @@ struct Partition { 4: i32 createTime, 5: i32 lastAccessTime, 6: StorageDescriptor sd, - 7: map parameters + 7: map parameters, + 8: optional PrincipalPrivilegeSet privileges } -// index on a hive table is also another table whose columns are the subset of the base table columns along with the offset -// this will automatically generate table (table_name_index_name) struct Index { 1: string indexName, // unique with in the whole database namespace - 2: i32 indexType, // reserved - 3: string tableName, - 4: string dbName, - 5: list colNames, // for now columns will be sorted in the ascending order - 6: string partName // partition name + 2: string indexHandlerClass, // reserved + 3: string dbName, + 4: string origTableName, + 5: i32 createTime, + 6: i32 lastAccessTime, + 7: string indexTableName, + 8: StorageDescriptor sd, + 9: map parameters, + 10: bool deferredRebuild } // schema of the table/query results etc. @@ -118,7 +181,6 @@ struct Schema { 2: map properties } - exception MetaException { 1: string message } @@ -160,16 +222,17 @@ exception ConfigValSecurityException { */ service ThriftHiveMetastore extends fb303.FacebookService { - bool create_database(1:string name, 2:string description) - throws(1:AlreadyExistsException o1, 2:MetaException o2) + void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3) Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2) - bool drop_database(1:string name) throws(2:MetaException o2) - list get_databases() throws(1:MetaException o1) - + void drop_database(1:string name, 2:bool deleteData) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3) + list get_databases(1:string pattern) throws(1:MetaException o1) + list get_all_databases() throws(1:MetaException o1) + void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2) + // returns the type with given name (make seperate calls for the dependent types if needed) - Type get_type(1:string name) throws(1:MetaException o2) + Type get_type(1:string name) throws(1:MetaException o1, 2:NoSuchObjectException o2) bool create_type(1:Type type) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3) - bool drop_type(1:string type) throws(1:MetaException o2) + bool drop_type(1:string type) throws(1:MetaException o1, 2:NoSuchObjectException o2) map get_type_all(1:string name) throws(1:MetaException o2) @@ -187,38 +250,77 @@ service ThriftHiveMetastore extends fb303.FacebookService // sd.inputFormat (SequenceFileInputFormat (binary like falcon tables or u_full) or TextInputFormat) // sd.outputFormat (SequenceFileInputFormat (binary) or TextInputFormat) // sd.serdeInfo.serializationLib (SerDe class name eg org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe + // * See notes on DDL_TIME void create_table(1:Table tbl) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4) // drops the table and all the partitions associated with it if the table has partitions // delete data (including partitions) if deleteData is set to true void drop_table(1:string dbname, 2:string name, 3:bool deleteData) throws(1:NoSuchObjectException o1, 2:MetaException o3) - list get_tables(1: string db_name, 2: string pattern) - throws (1: MetaException o1) + list get_tables(1: string db_name, 2: string pattern) throws (1: MetaException o1) + list get_all_tables(1: string db_name) throws (1: MetaException o1) Table get_table(1:string dbname, 2:string tbl_name) throws (1:MetaException o1, 2:NoSuchObjectException o2) // alter table applies to only future partitions not for existing partitions + // * See notes on DDL_TIME void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl) throws (1:InvalidOperationException o1, 2:MetaException o2) // the following applies to only tables that have partitions + // * See notes on DDL_TIME Partition add_partition(1:Partition new_part) throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3) Partition append_partition(1:string db_name, 2:string tbl_name, 3:list part_vals) throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3) + Partition append_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name) + throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3) bool drop_partition(1:string db_name, 2:string tbl_name, 3:list part_vals, 4:bool deleteData) throws(1:NoSuchObjectException o1, 2:MetaException o2) + bool drop_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name, 4:bool deleteData) + throws(1:NoSuchObjectException o1, 2:MetaException o2) Partition get_partition(1:string db_name, 2:string tbl_name, 3:list part_vals) - throws(1:MetaException o1) + throws(1:MetaException o1, 2:NoSuchObjectException o2) + + Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list part_vals, + 4: string user_name, 5: list group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2) + + Partition get_partition_by_name(1:string db_name 2:string tbl_name, 3:string part_name) + throws(1:MetaException o1, 2:NoSuchObjectException o2) + // returns all the partitions for this table in reverse chronological order. - // if max parts is given then it will return only that many + // If max parts is given then it will return only that many. list get_partitions(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1) throws(1:NoSuchObjectException o1, 2:MetaException o2) + list get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1, + 4: string user_name, 5: list group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2) + list get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1) throws(1:MetaException o2) + + // get_partition*_ps methods allow filtering by a partial partition specification, + // as needed for dynamic partitions. The values that are not restricted should + // be empty strings. Nulls were considered (instead of "") but caused errors in + // generated Python code. The size of part_vals may be smaller than the + // number of partition columns - the unspecified values are considered the same + // as "". + list get_partitions_ps(1:string db_name 2:string tbl_name + 3:list part_vals, 4:i16 max_parts=-1) + throws(1:MetaException o1) + list get_partitions_ps_with_auth(1:string db_name, 2:string tbl_name, 3:list part_vals, 4:i16 max_parts=-1, + 5: string user_name, 6: list group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2) + + list get_partition_names_ps(1:string db_name, + 2:string tbl_name, 3:list part_vals, 4:i16 max_parts=-1) + throws(1:MetaException o1) + + // get the partitions matching the given partition filter + list get_partitions_by_filter(1:string db_name 2:string tbl_name + 3:string filter, 4:i16 max_parts=-1) + throws(1:MetaException o1, 2:NoSuchObjectException o2) // changes the partition to the new partition object. partition is identified from the part values // in the new_part + // * See notes on DDL_TIME void alter_partition(1:string db_name, 2:string tbl_name, 3:Partition new_part) throws(1:InvalidOperationException o1, 2:MetaException o2) @@ -228,8 +330,81 @@ service ThriftHiveMetastore extends fb303.FacebookService // thrown. string get_config_value(1:string name, 2:string defaultValue) throws(1:ConfigValSecurityException o1) + + // converts a partition name into a partition values array + list partition_name_to_vals(1: string part_name) + throws(1: MetaException o1) + // converts a partition name into a partition specification (a mapping from + // the partition cols to the values) + map partition_name_to_spec(1: string part_name) + throws(1: MetaException o1) + + //index + Index add_index(1:Index new_index, 2: Table index_table) + throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3) + void alter_index(1:string dbname, 2:string base_tbl_name, 3:string idx_name, 4:Index new_idx) + throws (1:InvalidOperationException o1, 2:MetaException o2) + bool drop_index_by_name(1:string db_name, 2:string tbl_name, 3:string index_name, 4:bool deleteData) + throws(1:NoSuchObjectException o1, 2:MetaException o2) + Index get_index_by_name(1:string db_name 2:string tbl_name, 3:string index_name) + throws(1:MetaException o1, 2:NoSuchObjectException o2) + + list get_indexes(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1) + throws(1:NoSuchObjectException o1, 2:MetaException o2) + list get_index_names(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1) + throws(1:MetaException o2) + + //authorization privileges + + bool create_role(1:Role role) throws(1:MetaException o1) + bool drop_role(1:string role_name) throws(1:MetaException o1) + list get_role_names() throws(1:MetaException o1) + bool grant_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type, + 4:string grantor, 5:PrincipalType grantorType, 6:bool grant_option) throws(1:MetaException o1) + bool revoke_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type) + throws(1:MetaException o1) + list list_roles(1:string principal_name, 2:PrincipalType principal_type) throws(1:MetaException o1) + + PrincipalPrivilegeSet get_privilege_set(1:HiveObjectRef hiveObject, 2:string user_name, + 3: list group_names) throws(1:MetaException o1) + list list_privileges(1:string principal_name, 2:PrincipalType principal_type, + 3: HiveObjectRef hiveObject) throws(1:MetaException o1) + + bool grant_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1) + bool revoke_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1) + + //Authentication (delegation token) interfaces + + // get metastore server delegation token for use from the map/reduce tasks to authenticate + // to metastore server + string get_delegation_token(1:string renewer_kerberos_principal_name) throws (1:MetaException o1) + + // get metastore server delegation token for use from the map/reduce tasks to authenticate + // to metastore server - this method takes an extra token signature string which is just + // an identifier to associate with the token - this will be used by the token selector code + // to pick the right token given the associated identifier. + string get_delegation_token_with_signature(1:string renewer_kerberos_principal_name, + 2:string token_signature) throws (1:MetaException o1) + + // method to renew delegation token obtained from metastore server + i64 renew_delegation_token(1:string token_str_form) throws (1:MetaException o1) + + // method to cancel delegation token obtained from metastore server + void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1) } +// * Note about the DDL_TIME: When creating or altering a table or a partition, +// if the DDL_TIME is not set, the current time will be used. + +// For storing info about archived partitions in parameters + +// Whether the partition is archived +const string IS_ARCHIVED = "is_archived", +// The original location of the partition, before archiving. After archiving, +// this directory will contain the archive. When the partition +// is dropped, this directory will be deleted +const string ORIGINAL_LOCATION = "original_location", + // these should be needed only for backward compatibility with filestore const string META_TABLE_COLUMNS = "columns", const string META_TABLE_COLUMN_TYPES = "columns.types", @@ -243,5 +418,7 @@ const string META_TABLE_SERDE = "serde", const string META_TABLE_PARTITION_COLUMNS = "partition_columns", const string FILE_INPUT_FORMAT = "file.inputformat", const string FILE_OUTPUT_FORMAT = "file.outputformat", +const string META_TABLE_STORAGE = "storage_handler", +