diff --git a/comanage_utils.py b/comanage_utils.py index 238b0ec..7f5b5b9 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -6,7 +6,8 @@ import time import urllib.error import urllib.request -from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC +from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls +from dataclasses import dataclass #PRODUCTION VALUES @@ -16,6 +17,7 @@ PRODUCTION_OSG_CO_ID = 7 PRODUCTION_UNIX_CLUSTER_ID = 1 PRODUCTION_LDAP_TARGET_ID = 6 +LDAP_BASE_DN = "o=OSG,o=CO,dc=cilogon,dc=org" #TEST VALUES @@ -69,6 +71,14 @@ def mkauthstr(user, passwd): return encodebytes(raw_authstr.encode()).decode().replace("\n", "") +def get_ldap_authtok(ldap_authfile): + if ldap_authfile is not None: + ldap_authtok = open(ldap_authfile).readline().strip() + else: + raise PermissionError + return ldap_authtok + + def mkrequest(method, target, data, endpoint, authstr, **kw): url = os.path.join(endpoint, target) if kw: @@ -165,16 +175,43 @@ def get_datalist(data, listname): return data[listname] if data else [] +class LDAPSearch: + """ Wrapper class for LDAP searches. """ + server: Server = None + connection: Connection = None + + def __init__(self, ldap_server, ldap_user, ldap_authtok): + self.server = Server(ldap_server, get_info=ALL) + self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + + def search(self, ou, filter_str, attrs): + _, _, response, _ = self.connection.search(f"ou={ou},{LDAP_BASE_DN}", filter_str, attributes=attrs) + return response + def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): ldap_group_osggids = set() - server = Server(ldap_server, get_info=ALL) - connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES) + searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) + response = searcher.search("groups", "(cn=*)", ["gidNumber"]) for group in response: ldap_group_osggids.add(group["attributes"]["gidNumber"]) return ldap_group_osggids +def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): + """ Retrieve a dictionary of active users from LDAP, with their group memberships. """ + ldap_active_users = dict() + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + + searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) + response = searcher.search("people", filter_str, ["employeeNumber", "isMemberOf"]) + + for person in response: + ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", []) + + return ldap_active_users + + def identifier_from_list(id_list, id_type): id_type_list = [id["Type"] for id in id_list] try: diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 7d32bf0..b98b797 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -4,28 +4,37 @@ import re import sys import getopt -import collections +import requests import comanage_utils as utils SCRIPT = os.path.basename(__file__) -ENDPOINT = "https://registry.cilogon.org/registry/" -OSG_CO_ID = 7 +ENDPOINT = "https://registry-test.cilogon.org/registry/" +TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" +LDAP_SERVER = "ldaps://ldap-test.cilogon.org" +LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" +OSG_CO_ID = 8 +CACHE_FILENAME = "COmanage_Projects_cache.txt" +CACHE_LIFETIME_HOURS = 0.5 _usage = f"""\ -usage: [PASS=...] {SCRIPT} [OPTIONS] +usage: {SCRIPT} [OPTIONS] OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -s LDAP_SERVER specify LDAP server to read data from + -l LDAP_USER specify LDAP user for reading data from LDAP server + -a ldap_authfile specify path to file to open and read LDAP authtok -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint (default = {ENDPOINT}) -o outfile specify output file (default: write to stdout) -g filter_group filter users by group name (eg, 'ap1-login') - -l localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -n min_users Specify minimum number of users required to update the output file (default: 100) -h display this help text PASS for USER is taken from the first of: @@ -49,7 +58,11 @@ class Options: osg_co_id = OSG_CO_ID outfile = None authstr = None + ldap_server = LDAP_SERVER + ldap_user = LDAP_USER + ldap_authtok = None filtergrp = None + min_users = 100 # Bail out before updating the file if we have fewer than this many users localmaps = [] @@ -62,35 +75,12 @@ def get_osg_co_groups__map(): #print("get_osg_co_groups__map()") resp_data = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) data = utils.get_datalist(resp_data, "CoGroups") - return { g["Id"]: g["Name"] for g in data } - - -def co_group_is_ospool(gid): - #print(f"co_group_is_ospool({gid})") - resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) - data = utils.get_datalist(resp_data, "Identifiers") - return any( i["Type"] == "ospoolproject" for i in data ) - - -def get_co_group_members__pids(gid): - #print(f"get_co_group_members__pids({gid})") - resp_data = utils.get_co_group_members(gid, options.endpoint, options.authstr) - data = utils.get_datalist(resp_data, "CoGroupMembers") - # For INF-1060: Temporary Fix until "The Great Project Provisioning" is finished - return [ m["Person"]["Id"] for m in data if m["Member"] == True] - - -def get_co_person_osguser(pid): - #print(f"get_co_person_osguser({pid})") - resp_data = utils.get_co_person_identifiers(pid, options.endpoint, options.authstr) - data = utils.get_datalist(resp_data, "Identifiers") - typemap = { i["Type"]: i["Identifier"] for i in data } - return typemap.get("osguser") + return { g["Name"]: g["Id"] for g in data } def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:l:h') + ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h:n:m') except getopt.GetoptError: usage() @@ -99,21 +89,27 @@ def parse_options(args): passfd = None passfile = None + ldap_authfile = None for op, arg in ops: if op == '-h': usage() if op == '-u': options.user = arg if op == '-c': options.osg_co_id = int(arg) + if op == '-s': options.ldap_server= arg + if op == '-l': options.ldap_user = arg + if op == '-a': ldap_authfile = arg if op == '-d': passfd = int(arg) if op == '-f': passfile = arg if op == '-e': options.endpoint = arg if op == '-o': options.outfile = arg if op == '-g': options.filtergrp = arg - if op == '-l': options.localmaps = arg.split(",") + if op == '-m': options.localmaps = arg.split(",") + if op == '-n': options.min_users = int(arg) try: user, passwd = utils.getpw(options.user, passfd, passfile) options.authstr = utils.mkauthstr(user, passwd) + options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile) except PermissionError: usage("PASS required") @@ -123,36 +119,18 @@ def _deduplicate_list(items): """ return list(dict.fromkeys(items)) -def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): - pid_gids = collections.defaultdict(list) - - for gid in gid_pids: - for pid in gid_pids[gid]: - if pid_osguser[pid] is not None and gid not in pid_gids[pid]: - pid_gids[pid].append(gid) - - return pid_gids - - -def filter_by_group(pid_gids, groups, filter_group_name): - groups_idx = { v: k for k,v in groups.items() } - filter_gid = groups_idx[filter_group_name] # raises KeyError if missing - filter_group_pids = set(get_co_group_members__pids(filter_gid)) - return { p: g for p,g in pid_gids.items() if p in filter_group_pids } - - def get_osguser_groups(filter_group_name=None): - groups = get_osg_co_groups__map() - ospool_gids = filter(co_group_is_ospool, groups) - gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids } - all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] ) - pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids } - pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser) - if filter_group_name is not None: - pid_gids = filter_by_group(pid_gids, groups, filter_group_name) - - return { pid_osguser[pid]: map(groups.get, gids) - for pid, gids in pid_gids.items() } + ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) + topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() + project_names = topology_projects.keys() + + # Get COManage group IDs to preserve ordering from pre-LDAP migration script behavior + groups_ids = get_osg_co_groups__map() + return { + user: sorted([g for g in groups if g in project_names], key = lambda g: groups_ids.get(g, 0)) + for user, groups in ldap_users.items() + if any(g in project_names for g in groups) + } def parse_localmap(inputfile): @@ -204,6 +182,9 @@ def main(args): maps.append(parse_localmap(localmap)) osguser_groups_merged = merge_maps(maps) + # Sanity check, confirm we have generated a "sane" amount of user -> group mappings + if len(osguser_groups_merged) < options.min_users: + raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found") print_usermap(osguser_groups_merged)