From 898f83b64510e7a2aaa404f983f921bcfc3f0700 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 22 Nov 2023 09:36:34 -0600 Subject: [PATCH 01/18] Initial Commit for the new project-usermap script Uses LDAP search to find active and provisioned groups then compares them to the list project groups in COmanage to determine which LDAP groups and users to build the map out of. --- osg-comanage-project-usermap.py | 152 +++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 12 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 879a255..668a9fc 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 +import re import os import sys import json import getopt +import subprocess import collections import urllib.error import urllib.request @@ -16,6 +18,36 @@ MAXTIMEOUT = 625 TIMEOUTMULTIPLE = 5 +LDAP_AUTH_COMMAND = [ + "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", +] + +LDAP_GROUP_MEMBERS_COMMAND = [ + "ldapsearch", + "-H", + "ldaps://ldap.cilogon.org", + "-D", + "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-w", "{}", + "-b", + "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", + "-s", + "one", + "(cn=*)", +] + +LDAP_ACTIVE_USERS_COMMAND = [ + "ldapsearch", + "-LLL", + "-H", "ldaps://ldap.cilogon.org", + "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-x", + "-w", "{}", + "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", + "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "|", "grep", "voPersonApplicationUID", + "|", "sort", +] _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -142,13 +174,19 @@ def get_osg_co_groups__map(): return { g["Id"]: g["Name"] for g in data } -def co_group_is_ospool(gid): +def co_group_is_project(gid): #print(f"co_group_is_ospool({gid})") resp_data = get_co_group_identifiers(gid) data = get_datalist(resp_data, "Identifiers") return any( i["Type"] == "ospoolproject" for i in data ) +def get_co_group_osggid(gid): + resp_data = get_co_group_identifiers(gid) + data = get_datalist(resp_data, "Identifiers") + return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] + + def get_co_group_members__pids(gid): #print(f"get_co_group_members__pids({gid})") resp_data = get_co_group_members(gid) @@ -192,6 +230,84 @@ def parse_options(args): options.authstr = mkauthstr(user, passwd) +def get_ldap_group_members_data(): + gidNumber_str = "gidNumber: " + gidNumber_regex = re.compile(gidNumber_str) + member_str = f"hasMember: " + member_regex = re.compile(member_str) + + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + + data_file = subprocess.run( + ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + + search_results = list(filter( + lambda x: not re.compile("#|dn|cn|objectClass").match(x), + (line for line in data_file))) + + search_results.reverse() + + group_data_dict = dict() + index = 0 + while index < len(search_results) - 1: + while not gidNumber_regex.match(search_results[index]): + index += 1 + gid = search_results[index].replace(gidNumber_str, "") + members_list = [] + while search_results[index] != "": + if member_regex.match(search_results[index]): + members_list.append(search_results[index].replace(member_str, "")) + index += 1 + group_data_dict[gid] = members_list + index += 1 + + return group_data_dict + + +def get_ldap_active_users(): + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + + active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + return users + + +def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names): + users_to_projects_map = dict() + for osggid in project_to_user_map: + for user in project_to_user_map[osggid]: + if user in active_users: + if user not in users_to_projects_map: + users_to_projects_map[user] = [osggids_to_names[osggid]] + else: + users_to_projects_map[user].append(osggids_to_names[osggid]) + + return users_to_projects_map + + +def get_co_api_data(): + #TODO add cacheing for COManage API data + + groups = get_osg_co_groups__map() + project_osggids_to_name = dict() + for id,name in groups.items(): + if co_group_is_project(id): + project_osggids_to_name[get_co_group_osggid(id)] = name + return project_osggids_to_name + + def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): pid_gids = collections.defaultdict(set) @@ -211,17 +327,29 @@ def filter_by_group(pid_gids, groups, filter_group_name): def get_osguser_groups(filter_group_name=None): - groups = get_osg_co_groups__map() - ospool_gids = filter(co_group_is_ospool, groups) - gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids } - all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] ) - pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids } - pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser) - if filter_group_name is not None: - pid_gids = filter_by_group(pid_gids, groups, filter_group_name) - - return { pid_osguser[pid]: sorted(map(groups.get, gids)) - for pid, gids in pid_gids.items() } + project_osggids_to_name = get_co_api_data() + ldap_groups_members = get_ldap_group_members_data() + ldap_users = get_ldap_active_users() + + active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) + project_to_user_map = { + osggid : ldap_groups_members[osggid] + for osggid in active_project_osggids + } + all_project_users = set( + username for osggid in project_to_user_map for username in project_to_user_map[osggid] + ) + all_active_project_users = all_project_users.intersection(ldap_users) + usernames_to_project_map = create_user_to_projects_map( + project_to_user_map, + all_active_project_users, + project_osggids_to_name, + ) + + #if filter_group_name is not None: + #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) + + return usernames_to_project_map def print_usermap_to_file(osguser_groups, file): From 2ffdc5d260958686590a155ab7d61c51a53ce939 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 22 Nov 2023 12:01:52 -0600 Subject: [PATCH 02/18] Remove f string with no placeholders --- osg-comanage-project-usermap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 668a9fc..8b60738 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -233,7 +233,7 @@ def parse_options(args): def get_ldap_group_members_data(): gidNumber_str = "gidNumber: " gidNumber_regex = re.compile(gidNumber_str) - member_str = f"hasMember: " + member_str = "hasMember: " member_regex = re.compile(member_str) auth_str = subprocess.run( From 3efe8a66bdc749b5010ebb443aceb0d46bf79538 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 27 Nov 2023 13:55:12 -0600 Subject: [PATCH 03/18] Reimplement filtering by members of a group And clean up of no-longer-used methods remove collections import --- osg-comanage-project-usermap.py | 41 +++++++++------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 8b60738..fd02617 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -6,7 +6,6 @@ import json import getopt import subprocess -import collections import urllib.error import urllib.request @@ -28,7 +27,7 @@ "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-w", "{}", + "-w", "{auth}", "-b", "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "-s", @@ -42,9 +41,9 @@ "-H", "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", "-x", - "-w", "{}", + "-w", "{auth}", "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", - "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "{filter}", "voPersonApplicationUID", "|", "grep", "voPersonApplicationUID", "|", "sort", ] @@ -242,13 +241,13 @@ def get_ldap_group_members_data(): ).stdout.decode('utf-8').strip() ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND - ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str data_file = subprocess.run( ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') search_results = list(filter( - lambda x: not re.compile("#|dn|cn|objectClass").match(x), + lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x), (line for line in data_file))) search_results.reverse() @@ -270,14 +269,17 @@ def get_ldap_group_members_data(): return group_data_dict -def get_ldap_active_users(): +def get_ldap_active_users(filter_group_name): auth_str = subprocess.run( LDAP_AUTH_COMMAND, stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() + + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) @@ -308,28 +310,10 @@ def get_co_api_data(): return project_osggids_to_name -def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): - pid_gids = collections.defaultdict(set) - - for gid in gid_pids: - for pid in gid_pids[gid]: - if pid_osguser[pid] is not None: - pid_gids[pid].add(gid) - - return pid_gids - - -def filter_by_group(pid_gids, groups, filter_group_name): - groups_idx = { v: k for k,v in groups.items() } - filter_gid = groups_idx[filter_group_name] # raises KeyError if missing - filter_group_pids = set(get_co_group_members__pids(filter_gid)) - return { p: g for p,g in pid_gids.items() if p in filter_group_pids } - - def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() - ldap_users = get_ldap_active_users() + ldap_users = get_ldap_active_users(filter_group_name) active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) project_to_user_map = { @@ -345,9 +329,6 @@ def get_osguser_groups(filter_group_name=None): all_active_project_users, project_osggids_to_name, ) - - #if filter_group_name is not None: - #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) return usernames_to_project_map From 895775321bcd0e3d1d663face70ba07b38e64692 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 27 Nov 2023 15:59:28 -0600 Subject: [PATCH 04/18] Add caching for COmanage API data Writes the osggids to project name dict to a file containing the epoch time the cache was made. If the file is found to exist and was made in the past 0.5 hours (by default), the program will read from the cache instead of making COmanage API calls. Otherwise, the program will get the project data from the API and overwrite any existing cache. --- osg-comanage-project-usermap.py | 39 ++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index fd02617..4570dd7 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -4,6 +4,7 @@ import os import sys import json +import time import getopt import subprocess import urllib.error @@ -16,6 +17,8 @@ MINTIMEOUT = 5 MAXTIMEOUT = 625 TIMEOUTMULTIPLE = 5 +CACHE_FILENAME = "COmanage_Projects_cache.txt" +CACHE_LIFETIME_HOURS = 0.5 LDAP_AUTH_COMMAND = [ "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", @@ -275,14 +278,16 @@ def get_ldap_active_users(filter_group_name): stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() - filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") + else "" for line in active_users) return users @@ -299,9 +304,7 @@ def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_na return users_to_projects_map -def get_co_api_data(): - #TODO add cacheing for COManage API data - +def get_groups_data_from_api(): groups = get_osg_co_groups__map() project_osggids_to_name = dict() for id,name in groups.items(): @@ -310,6 +313,32 @@ def get_co_api_data(): return project_osggids_to_name +def get_co_api_data(): + try: + r = open(CACHE_FILENAME, "r") + lines = r.readlines() + if float(lines[0]) >= (time.time() - (60 * 60 * CACHE_LIFETIME_HOURS)): + entries = lines[1:len(lines)] + project_osggids_to_name = dict() + for entry in entries: + osggid_name_pair = entry.split(":") + if len(osggid_name_pair) == 2: + project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1] + else: + raise OSError + except OSError: + with open(CACHE_FILENAME, "w") as w: + project_osggids_to_name = get_groups_data_from_api() + print(time.time(), file=w) + for osggid, name in project_osggids_to_name.items(): + print(f"{osggid}:{name}", file=w) + finally: + if r: + r.close() + + return project_osggids_to_name + + def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() From a4d64779f58b064ce58a8502b0278bff493b8fe0 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:18:55 -0600 Subject: [PATCH 05/18] Initial Commit for the new project-usermap script Uses LDAP search to find active and provisioned groups then compares them to the list project groups in COmanage to determine which LDAP groups and users to build the map out of. --- osg-comanage-project-usermap.py | 152 +++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 12 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 04b5983..7bbfb11 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import re import os import sys import getopt +import subprocess import collections import urllib.error import urllib.request @@ -13,6 +15,36 @@ ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 +LDAP_AUTH_COMMAND = [ + "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", +] + +LDAP_GROUP_MEMBERS_COMMAND = [ + "ldapsearch", + "-H", + "ldaps://ldap.cilogon.org", + "-D", + "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-w", "{}", + "-b", + "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", + "-s", + "one", + "(cn=*)", +] + +LDAP_ACTIVE_USERS_COMMAND = [ + "ldapsearch", + "-LLL", + "-H", "ldaps://ldap.cilogon.org", + "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-x", + "-w", "{}", + "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", + "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "|", "grep", "voPersonApplicationUID", + "|", "sort", +] _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -64,13 +96,19 @@ def get_osg_co_groups__map(): return { g["Id"]: g["Name"] for g in data } -def co_group_is_ospool(gid): +def co_group_is_project(gid): #print(f"co_group_is_ospool({gid})") resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) data = utils.get_datalist(resp_data, "Identifiers") return any( i["Type"] == "ospoolproject" for i in data ) +def get_co_group_osggid(gid): + resp_data = get_co_group_identifiers(gid) + data = get_datalist(resp_data, "Identifiers") + return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] + + def get_co_group_members__pids(gid): #print(f"get_co_group_members__pids({gid})") resp_data = utils.get_co_group_members(gid, options.endpoint, options.authstr) @@ -115,6 +153,84 @@ def parse_options(args): usage("PASS required") +def get_ldap_group_members_data(): + gidNumber_str = "gidNumber: " + gidNumber_regex = re.compile(gidNumber_str) + member_str = f"hasMember: " + member_regex = re.compile(member_str) + + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + + data_file = subprocess.run( + ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + + search_results = list(filter( + lambda x: not re.compile("#|dn|cn|objectClass").match(x), + (line for line in data_file))) + + search_results.reverse() + + group_data_dict = dict() + index = 0 + while index < len(search_results) - 1: + while not gidNumber_regex.match(search_results[index]): + index += 1 + gid = search_results[index].replace(gidNumber_str, "") + members_list = [] + while search_results[index] != "": + if member_regex.match(search_results[index]): + members_list.append(search_results[index].replace(member_str, "")) + index += 1 + group_data_dict[gid] = members_list + index += 1 + + return group_data_dict + + +def get_ldap_active_users(): + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + + active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + return users + + +def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names): + users_to_projects_map = dict() + for osggid in project_to_user_map: + for user in project_to_user_map[osggid]: + if user in active_users: + if user not in users_to_projects_map: + users_to_projects_map[user] = [osggids_to_names[osggid]] + else: + users_to_projects_map[user].append(osggids_to_names[osggid]) + + return users_to_projects_map + + +def get_co_api_data(): + #TODO add cacheing for COManage API data + + groups = get_osg_co_groups__map() + project_osggids_to_name = dict() + for id,name in groups.items(): + if co_group_is_project(id): + project_osggids_to_name[get_co_group_osggid(id)] = name + return project_osggids_to_name + + def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): pid_gids = collections.defaultdict(set) @@ -134,17 +250,29 @@ def filter_by_group(pid_gids, groups, filter_group_name): def get_osguser_groups(filter_group_name=None): - groups = get_osg_co_groups__map() - ospool_gids = filter(co_group_is_ospool, groups) - gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids } - all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] ) - pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids } - pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser) - if filter_group_name is not None: - pid_gids = filter_by_group(pid_gids, groups, filter_group_name) - - return { pid_osguser[pid]: sorted(map(groups.get, gids)) - for pid, gids in pid_gids.items() } + project_osggids_to_name = get_co_api_data() + ldap_groups_members = get_ldap_group_members_data() + ldap_users = get_ldap_active_users() + + active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) + project_to_user_map = { + osggid : ldap_groups_members[osggid] + for osggid in active_project_osggids + } + all_project_users = set( + username for osggid in project_to_user_map for username in project_to_user_map[osggid] + ) + all_active_project_users = all_project_users.intersection(ldap_users) + usernames_to_project_map = create_user_to_projects_map( + project_to_user_map, + all_active_project_users, + project_osggids_to_name, + ) + + #if filter_group_name is not None: + #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) + + return usernames_to_project_map def print_usermap_to_file(osguser_groups, file): From da46112e395731a7dd88c7b220a41436b63efef9 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:18:55 -0600 Subject: [PATCH 06/18] Remove f string with no placeholders --- osg-comanage-project-usermap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 7bbfb11..4fb150b 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -156,7 +156,7 @@ def parse_options(args): def get_ldap_group_members_data(): gidNumber_str = "gidNumber: " gidNumber_regex = re.compile(gidNumber_str) - member_str = f"hasMember: " + member_str = "hasMember: " member_regex = re.compile(member_str) auth_str = subprocess.run( From da9f34faf1ec5d93dd52b05bfff80c57da48906a Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:18:55 -0600 Subject: [PATCH 07/18] Reimplement filtering by members of a group And clean up of no-longer-used methods remove collections import --- osg-comanage-project-usermap.py | 41 +++++++++------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 4fb150b..963e952 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -5,7 +5,6 @@ import sys import getopt import subprocess -import collections import urllib.error import urllib.request import comanage_scripts_utils as utils @@ -25,7 +24,7 @@ "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-w", "{}", + "-w", "{auth}", "-b", "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "-s", @@ -39,9 +38,9 @@ "-H", "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", "-x", - "-w", "{}", + "-w", "{auth}", "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", - "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "{filter}", "voPersonApplicationUID", "|", "grep", "voPersonApplicationUID", "|", "sort", ] @@ -165,13 +164,13 @@ def get_ldap_group_members_data(): ).stdout.decode('utf-8').strip() ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND - ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str data_file = subprocess.run( ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') search_results = list(filter( - lambda x: not re.compile("#|dn|cn|objectClass").match(x), + lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x), (line for line in data_file))) search_results.reverse() @@ -193,14 +192,17 @@ def get_ldap_group_members_data(): return group_data_dict -def get_ldap_active_users(): +def get_ldap_active_users(filter_group_name): auth_str = subprocess.run( LDAP_AUTH_COMMAND, stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() + + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) @@ -231,28 +233,10 @@ def get_co_api_data(): return project_osggids_to_name -def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): - pid_gids = collections.defaultdict(set) - - for gid in gid_pids: - for pid in gid_pids[gid]: - if pid_osguser[pid] is not None: - pid_gids[pid].add(gid) - - return pid_gids - - -def filter_by_group(pid_gids, groups, filter_group_name): - groups_idx = { v: k for k,v in groups.items() } - filter_gid = groups_idx[filter_group_name] # raises KeyError if missing - filter_group_pids = set(get_co_group_members__pids(filter_gid)) - return { p: g for p,g in pid_gids.items() if p in filter_group_pids } - - def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() - ldap_users = get_ldap_active_users() + ldap_users = get_ldap_active_users(filter_group_name) active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) project_to_user_map = { @@ -268,9 +252,6 @@ def get_osguser_groups(filter_group_name=None): all_active_project_users, project_osggids_to_name, ) - - #if filter_group_name is not None: - #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) return usernames_to_project_map From 20555800b2b50a25504da44bb84dff80f6f14050 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:20:31 -0600 Subject: [PATCH 08/18] Add caching for COmanage API data Writes the osggids to project name dict to a file containing the epoch time the cache was made. If the file is found to exist and was made in the past 0.5 hours (by default), the program will read from the cache instead of making COmanage API calls. Otherwise, the program will get the project data from the API and overwrite any existing cache. --- osg-comanage-project-usermap.py | 43 +++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 963e952..2f641dc 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -3,6 +3,8 @@ import re import os import sys +import json +import time import getopt import subprocess import urllib.error @@ -13,6 +15,11 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 +MINTIMEOUT = 5 +MAXTIMEOUT = 625 +TIMEOUTMULTIPLE = 5 +CACHE_FILENAME = "COmanage_Projects_cache.txt" +CACHE_LIFETIME_HOURS = 0.5 LDAP_AUTH_COMMAND = [ "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", @@ -198,14 +205,16 @@ def get_ldap_active_users(filter_group_name): stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() - filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") + else "" for line in active_users) return users @@ -222,9 +231,7 @@ def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_na return users_to_projects_map -def get_co_api_data(): - #TODO add cacheing for COManage API data - +def get_groups_data_from_api(): groups = get_osg_co_groups__map() project_osggids_to_name = dict() for id,name in groups.items(): @@ -233,6 +240,32 @@ def get_co_api_data(): return project_osggids_to_name +def get_co_api_data(): + try: + r = open(CACHE_FILENAME, "r") + lines = r.readlines() + if float(lines[0]) >= (time.time() - (60 * 60 * CACHE_LIFETIME_HOURS)): + entries = lines[1:len(lines)] + project_osggids_to_name = dict() + for entry in entries: + osggid_name_pair = entry.split(":") + if len(osggid_name_pair) == 2: + project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1] + else: + raise OSError + except OSError: + with open(CACHE_FILENAME, "w") as w: + project_osggids_to_name = get_groups_data_from_api() + print(time.time(), file=w) + for osggid, name in project_osggids_to_name.items(): + print(f"{osggid}:{name}", file=w) + finally: + if r: + r.close() + + return project_osggids_to_name + + def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() From 3d78c61039ebb0d46c251856487665e675ef7853 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Fri, 29 Dec 2023 10:26:46 -0600 Subject: [PATCH 09/18] project-usermap requested changes --- comanage_scripts_utils.py | 35 ++++++++- osg-comanage-project-usermap.py | 130 ++++++-------------------------- 2 files changed, 58 insertions(+), 107 deletions(-) diff --git a/comanage_scripts_utils.py b/comanage_scripts_utils.py index 96c1c41..093aaa3 100644 --- a/comanage_scripts_utils.py +++ b/comanage_scripts_utils.py @@ -6,7 +6,7 @@ import json import urllib.error import urllib.request -from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC +from ldap3 import Server, Connection, ALL, SAFE_SYNC MIN_TIMEOUT = 5 @@ -42,6 +42,14 @@ def mkauthstr(user, passwd): return encodebytes(raw_authstr.encode()).decode().replace("\n", "") +def get_ldap_authtok(ldap_authfile): + if ldap_authfile is not None: + ldap_authtok = open(ldap_authfile).readline().rstrip("\n") + else: + raise PermissionError + return ldap_authtok + + def mkrequest(method, target, data, endpoint, authstr, **kw): url = os.path.join(endpoint, target) if kw: @@ -135,12 +143,35 @@ def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): ldap_group_osggids = set() server = Server(ldap_server, get_info=ALL) connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES) + _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=["gidNumber"]) for group in response: ldap_group_osggids.add(group["attributes"]["gidNumber"]) return ldap_group_osggids +def get_ldap_group_members(ldap_gid, ldap_server, ldap_user, ldap_authtok): + ldap_group_members = set() + server = Server(ldap_server, get_info=ALL) + connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", f"(&(gidNumber={ldap_gid})(cn=*))", attributes=["hasMember"]) + for group in response: + ldap_group_members.update(group["attributes"]["hasMember"]) + return ldap_group_members + + +def get_ldap_active_users(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): + ldap_active_users = set() + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + server = Server(ldap_server, get_info=ALL) + connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=people,o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=["employeeNumber"]) + for person in response: + # the "employeeNumber" is the person's name in the first.last format + ldap_active_users.add(person["attributes"]["employeeNumber"]) + return ldap_active_users + + def identifier_from_list(id_list, id_type): id_type_list = [id["Type"] for id in id_list] try: diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index cd8e96f..7f6eb67 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,12 +1,9 @@ #!/usr/bin/env python3 -import re import os import sys -import json import time import getopt -import subprocess import urllib.error import urllib.request import comanage_scripts_utils as utils @@ -14,43 +11,12 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" +LDAP_SERVER = "ldaps://ldap-test.cilogon.org" +LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 8 -MINTIMEOUT = 5 -MAXTIMEOUT = 625 -TIMEOUTMULTIPLE = 5 CACHE_FILENAME = "COmanage_Projects_cache.txt" CACHE_LIFETIME_HOURS = 0.5 -LDAP_AUTH_COMMAND = [ - "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", -] - -LDAP_GROUP_MEMBERS_COMMAND = [ - "ldapsearch", - "-H", - "ldaps://ldap.cilogon.org", - "-D", - "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-w", "{auth}", - "-b", - "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", - "-s", - "one", - "(cn=*)", -] - -LDAP_ACTIVE_USERS_COMMAND = [ - "ldapsearch", - "-LLL", - "-H", "ldaps://ldap.cilogon.org", - "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-x", - "-w", "{auth}", - "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", - "{filter}", "voPersonApplicationUID", - "|", "grep", "voPersonApplicationUID", - "|", "sort", -] _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -58,6 +24,9 @@ OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -s LDAP_SERVER specify LDAP server to read data from + -l LDAP_USER specify LDAP user for reading data from LDAP server + -a ldap_authfile specify path to file to open and read LDAP authtok -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint @@ -87,6 +56,9 @@ class Options: osg_co_id = OSG_CO_ID outfile = None authstr = None + ldap_server = LDAP_SERVER + ldap_user = LDAP_USER + ldap_authtok = None filtergrp = None @@ -110,14 +82,8 @@ def co_group_is_project(gid): def get_co_group_osggid(gid): - resp_data = get_co_group_identifiers(gid) - data = get_datalist(resp_data, "Identifiers") - return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] - - -def get_co_group_osggid(gid): - resp_data = get_co_group_identifiers(gid) - data = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "Identifiers") return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] @@ -138,7 +104,7 @@ def get_co_person_osguser(pid): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:h') + ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h') except getopt.GetoptError: usage() @@ -147,11 +113,15 @@ def parse_options(args): passfd = None passfile = None + ldap_authfile = None for op, arg in ops: if op == '-h': usage() if op == '-u': options.user = arg if op == '-c': options.osg_co_id = int(arg) + if op == '-s': options.ldap_server= arg + if op == '-l': options.ldap_user = arg + if op == '-a': ldap_authfile = arg if op == '-d': passfd = int(arg) if op == '-f': passfile = arg if op == '-e': options.endpoint = arg @@ -161,69 +131,20 @@ def parse_options(args): try: user, passwd = utils.getpw(options.user, passfd, passfile) options.authstr = utils.mkauthstr(user, passwd) + options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile) except PermissionError: usage("PASS required") -def get_ldap_group_members_data(): - gidNumber_str = "gidNumber: " - gidNumber_regex = re.compile(gidNumber_str) - member_str = "hasMember: " - member_regex = re.compile(member_str) - - auth_str = subprocess.run( - LDAP_AUTH_COMMAND, - stdout=subprocess.PIPE - ).stdout.decode('utf-8').strip() - - ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND - ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str - - data_file = subprocess.run( - ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - - search_results = list(filter( - lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x), - (line for line in data_file))) - - search_results.reverse() - +def get_ldap_group_members_dict(): group_data_dict = dict() - index = 0 - while index < len(search_results) - 1: - while not gidNumber_regex.match(search_results[index]): - index += 1 - gid = search_results[index].replace(gidNumber_str, "") - members_list = [] - while search_results[index] != "": - if member_regex.match(search_results[index]): - members_list.append(search_results[index].replace(member_str, "")) - index += 1 - group_data_dict[gid] = members_list - index += 1 + for group_gid in utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok): + group_members = utils.get_ldap_group_members(group_gid, options.ldap_server, options.ldap_user, options.ldap_authtok) + group_data_dict[group_gid] = group_members return group_data_dict -def get_ldap_active_users(filter_group_name): - auth_str = subprocess.run( - LDAP_AUTH_COMMAND, - stdout=subprocess.PIPE - ).stdout.decode('utf-8').strip() - - filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None - else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") - - ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str - - active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") - else "" for line in active_users) - return users - - def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names): users_to_projects_map = dict() for osggid in project_to_user_map: @@ -256,8 +177,10 @@ def get_co_api_data(): for entry in entries: osggid_name_pair = entry.split(":") if len(osggid_name_pair) == 2: - project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1] + project_osggids_to_name[int(osggid_name_pair[0])] = osggid_name_pair[1].strip() + r.close() else: + r.close() raise OSError except OSError: with open(CACHE_FILENAME, "w") as w: @@ -265,17 +188,14 @@ def get_co_api_data(): print(time.time(), file=w) for osggid, name in project_osggids_to_name.items(): print(f"{osggid}:{name}", file=w) - finally: - if r: - r.close() return project_osggids_to_name def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() - ldap_groups_members = get_ldap_group_members_data() - ldap_users = get_ldap_active_users(filter_group_name) + ldap_groups_members = get_ldap_group_members_dict() + ldap_users = utils.get_ldap_active_users(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) project_to_user_map = { From c410e7c70130934580d20ca255d1254f1dc163c6 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Mon, 17 Nov 2025 12:31:25 -0600 Subject: [PATCH 10/18] Get users and groups directly from ldap, query for group 'project-ness' from topology rather than comanage --- comanage_utils.py | 13 +++++++++++++ osg-comanage-project-usermap.py | 26 ++++++-------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index 13da333..a907b40 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -205,6 +205,19 @@ def get_ldap_active_users(ldap_server, ldap_user, ldap_authtok, filter_group_nam ldap_active_users.add(person["attributes"]["employeeNumber"]) return ldap_active_users +def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): + """ Retrieve a dictionary of active users from LDAP, with their group memberships. """ + ldap_active_users = {} + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + server = Server(ldap_server, get_info=ALL) + connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=people,o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=["employeeNumber", "isMemberOf"]) + for person in response: + ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", []) + + return ldap_active_users + def identifier_from_list(id_list, id_type): id_type_list = [id["Type"] for id in id_list] diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index e4e4e5a..ce9b60f 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -7,11 +7,13 @@ import getopt import urllib.error import urllib.request +import requests import comanage_utils as utils SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" +TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" LDAP_SERVER = "ldaps://ldap-test.cilogon.org" LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 8 @@ -203,26 +205,10 @@ def get_co_api_data(): def get_osguser_groups(filter_group_name=None): - project_osggids_to_name = get_co_api_data() - ldap_groups_members = get_ldap_group_members_dict() - ldap_users = utils.get_ldap_active_users(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) - - active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) - project_to_user_map = { - osggid : ldap_groups_members[osggid] - for osggid in active_project_osggids - } - all_project_users = set( - username for osggid in project_to_user_map for username in project_to_user_map[osggid] - ) - all_active_project_users = all_project_users.intersection(ldap_users) - usernames_to_project_map = create_user_to_projects_map( - project_to_user_map, - all_active_project_users, - project_osggids_to_name, - ) - - return usernames_to_project_map + ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) + topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() + project_names = topology_projects.keys() + return {user: [p for p in groups if p in project_names] for user, groups in ldap_users.items()} def parse_localmap(inputfile): From c340729eae56962100fa04a8ad7e1609483dca9b Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 18 Nov 2025 14:40:56 -0600 Subject: [PATCH 11/18] Filter out users with no associated projects --- osg-comanage-project-usermap.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index ce9b60f..bc9e2d3 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -22,7 +22,7 @@ _usage = f"""\ -usage: [PASS=...] {SCRIPT} [OPTIONS] +usage: {SCRIPT} [OPTIONS] OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line @@ -36,7 +36,7 @@ (default = {ENDPOINT}) -o outfile specify output file (default: write to stdout) -g filter_group filter users by group name (eg, 'ap1-login') - -l localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile -h display this help text PASS for USER is taken from the first of: @@ -133,7 +133,7 @@ def parse_options(args): if op == '-e': options.endpoint = arg if op == '-o': options.outfile = arg if op == '-g': options.filtergrp = arg - if op == '-l': options.localmaps = arg.split(",") + if op == '-m': options.localmaps = arg.split(",") try: user, passwd = utils.getpw(options.user, passfd, passfile) @@ -208,7 +208,11 @@ def get_osguser_groups(filter_group_name=None): ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() project_names = topology_projects.keys() - return {user: [p for p in groups if p in project_names] for user, groups in ldap_users.items()} + return { + user: [g for g in groups if g in project_names] + for user, groups in ldap_users.items() + if any(g in project_names for g in groups) + } def parse_localmap(inputfile): From 8c9a033efd875315a98c3fccac7d5b085b1bbfcc Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 18 Nov 2025 14:41:28 -0600 Subject: [PATCH 12/18] Filter out non-active users --- osg-comanage-project-usermap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index bc9e2d3..1312d44 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -212,6 +212,7 @@ def get_osguser_groups(filter_group_name=None): user: [g for g in groups if g in project_names] for user, groups in ldap_users.items() if any(g in project_names for g in groups) + and any(':members:active' in g for g in groups) } From e3677da2587c6e88ed53c8eb3f61ae25fea32d1c Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Wed, 19 Nov 2025 18:08:51 -0600 Subject: [PATCH 13/18] Query COManage to preserve ordering of groups with old behavior --- osg-comanage-project-usermap.py | 80 ++++++--------------------------- 1 file changed, 13 insertions(+), 67 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 1312d44..c017cd4 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -37,6 +37,7 @@ -o outfile specify output file (default: write to stdout) -g filter_group filter users by group name (eg, 'ap1-login') -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -n min_users Specify minimum number of users required to update the output file (default: 100) -h display this help text PASS for USER is taken from the first of: @@ -64,6 +65,7 @@ class Options: ldap_user = LDAP_USER ldap_authtok = None filtergrp = None + min_users = 100 # Bail out before updating the file if we have fewer than this many users localmaps = [] @@ -76,41 +78,12 @@ def get_osg_co_groups__map(): #print("get_osg_co_groups__map()") resp_data = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) data = utils.get_datalist(resp_data, "CoGroups") - return { g["Id"]: g["Name"] for g in data } - - -def co_group_is_project(gid): - #print(f"co_group_is_ospool({gid})") - resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) - data = utils.get_datalist(resp_data, "Identifiers") - return any( i["Type"] == "ospoolproject" for i in data ) - - -def get_co_group_osggid(gid): - resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) - data = utils.get_datalist(resp_data, "Identifiers") - return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] - - -def get_co_group_members__pids(gid): - #print(f"get_co_group_members__pids({gid})") - resp_data = utils.get_co_group_members(gid, options.endpoint, options.authstr) - data = utils.get_datalist(resp_data, "CoGroupMembers") - # For INF-1060: Temporary Fix until "The Great Project Provisioning" is finished - return [ m["Person"]["Id"] for m in data if m["Member"] == True] - - -def get_co_person_osguser(pid): - #print(f"get_co_person_osguser({pid})") - resp_data = utils.get_co_person_identifiers(pid, options.endpoint, options.authstr) - data = utils.get_datalist(resp_data, "Identifiers") - typemap = { i["Type"]: i["Identifier"] for i in data } - return typemap.get("osguser") + return { g["Name"]: g["Id"] for g in data } def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h') + ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h:n:m') except getopt.GetoptError: usage() @@ -133,7 +106,8 @@ def parse_options(args): if op == '-e': options.endpoint = arg if op == '-o': options.outfile = arg if op == '-g': options.filtergrp = arg - if op == '-m': options.localmaps = arg.split(",") + if op == '-m': options.localmaps = arg.split(",") + if op == '-n': options.min_users = int(arg) try: user, passwd = utils.getpw(options.user, passfd, passfile) @@ -170,46 +144,15 @@ def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_na return users_to_projects_map -def get_groups_data_from_api(): - groups = get_osg_co_groups__map() - project_osggids_to_name = dict() - for id,name in groups.items(): - if co_group_is_project(id): - project_osggids_to_name[get_co_group_osggid(id)] = name - return project_osggids_to_name - - -def get_co_api_data(): - try: - r = open(CACHE_FILENAME, "r") - lines = r.readlines() - if float(lines[0]) >= (time.time() - (60 * 60 * CACHE_LIFETIME_HOURS)): - entries = lines[1:len(lines)] - project_osggids_to_name = dict() - for entry in entries: - osggid_name_pair = entry.split(":") - if len(osggid_name_pair) == 2: - project_osggids_to_name[int(osggid_name_pair[0])] = osggid_name_pair[1].strip() - r.close() - else: - r.close() - raise OSError - except OSError: - with open(CACHE_FILENAME, "w") as w: - project_osggids_to_name = get_groups_data_from_api() - print(time.time(), file=w) - for osggid, name in project_osggids_to_name.items(): - print(f"{osggid}:{name}", file=w) - - return project_osggids_to_name - - def get_osguser_groups(filter_group_name=None): ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() project_names = topology_projects.keys() + + # Get COManage group IDs to preserve ordering from pre-LDAP migration script behavior + groups_ids = get_osg_co_groups__map() return { - user: [g for g in groups if g in project_names] + user: sorted([g for g in groups if g in project_names], key = lambda g: groups_ids.get(g, 0)) for user, groups in ldap_users.items() if any(g in project_names for g in groups) and any(':members:active' in g for g in groups) @@ -265,6 +208,9 @@ def main(args): maps.append(parse_localmap(localmap)) osguser_groups_merged = merge_maps(maps) + # Sanity check, confirm we have generated a "sane" amount of user -> group mappings + if len(osguser_groups_merged) < options.min_users: + raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found") print_usermap(osguser_groups_merged) From df440b4fbfaaa1e9364dda6bf2a4e822ff26aa92 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Wed, 26 Nov 2025 11:51:44 -0600 Subject: [PATCH 14/18] Remove unused code from first draft of PR --- comanage_utils.py | 26 ++------------------------ osg-comanage-project-usermap.py | 25 ------------------------- 2 files changed, 2 insertions(+), 49 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index a907b40..c1cfd04 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -6,7 +6,7 @@ import time import urllib.error import urllib.request -from ldap3 import Server, Connection, ALL, SAFE_SYNC +from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls #PRODUCTION VALUES @@ -71,7 +71,7 @@ def mkauthstr(user, passwd): def get_ldap_authtok(ldap_authfile): if ldap_authfile is not None: - ldap_authtok = open(ldap_authfile).readline().rstrip("\n") + ldap_authtok = open(ldap_authfile).readline().strip() else: raise PermissionError return ldap_authtok @@ -183,28 +183,6 @@ def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): return ldap_group_osggids -def get_ldap_group_members(ldap_gid, ldap_server, ldap_user, ldap_authtok): - ldap_group_members = set() - server = Server(ldap_server, get_info=ALL) - connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", f"(&(gidNumber={ldap_gid})(cn=*))", attributes=["hasMember"]) - for group in response: - ldap_group_members.update(group["attributes"]["hasMember"]) - return ldap_group_members - - -def get_ldap_active_users(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): - ldap_active_users = set() - filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None - else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") - server = Server(ldap_server, get_info=ALL) - connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=people,o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=["employeeNumber"]) - for person in response: - # the "employeeNumber" is the person's name in the first.last format - ldap_active_users.add(person["attributes"]["employeeNumber"]) - return ldap_active_users - def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): """ Retrieve a dictionary of active users from LDAP, with their group memberships. """ ldap_active_users = {} diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index c017cd4..b62bd3a 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -3,10 +3,7 @@ import os import re import sys -import time import getopt -import urllib.error -import urllib.request import requests import comanage_utils as utils @@ -122,28 +119,6 @@ def _deduplicate_list(items): """ return list(dict.fromkeys(items)) -def get_ldap_group_members_dict(): - group_data_dict = dict() - for group_gid in utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok): - group_members = utils.get_ldap_group_members(group_gid, options.ldap_server, options.ldap_user, options.ldap_authtok) - group_data_dict[group_gid] = group_members - - return group_data_dict - - -def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names): - users_to_projects_map = dict() - for osggid in project_to_user_map: - for user in project_to_user_map[osggid]: - if user in active_users: - if user not in users_to_projects_map: - users_to_projects_map[user] = [osggids_to_names[osggid]] - else: - users_to_projects_map[user].append(osggids_to_names[osggid]) - - return users_to_projects_map - - def get_osguser_groups(filter_group_name=None): ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() From 4b3aca6551a23a5a97e4da1455c2d05d727bbf6b Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Wed, 26 Nov 2025 13:51:02 -0600 Subject: [PATCH 15/18] remove redundant check for active group membership --- osg-comanage-project-usermap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index b62bd3a..b98b797 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -130,7 +130,6 @@ def get_osguser_groups(filter_group_name=None): user: sorted([g for g in groups if g in project_names], key = lambda g: groups_ids.get(g, 0)) for user, groups in ldap_users.items() if any(g in project_names for g in groups) - and any(':members:active' in g for g in groups) } From 784c1ab797ff731716b6c121cc826c3f20b05177 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Wed, 26 Nov 2025 13:52:48 -0600 Subject: [PATCH 16/18] Use explicit dict rather than {} --- comanage_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comanage_utils.py b/comanage_utils.py index c1cfd04..b138fe9 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -185,7 +185,7 @@ def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): """ Retrieve a dictionary of active users from LDAP, with their group memberships. """ - ldap_active_users = {} + ldap_active_users = dict() filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") server = Server(ldap_server, get_info=ALL) From 0befc749863f0e702bd3c390a17dbfbda81ffbd6 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Wed, 26 Nov 2025 16:58:19 -0600 Subject: [PATCH 17/18] Refactor shared ldap connection construction logic out of get_ldap_groups/get_ldap_active_users_and_groups --- comanage_utils.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/comanage_utils.py b/comanage_utils.py index b138fe9..0e9f4f0 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -7,6 +7,7 @@ import urllib.error import urllib.request from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls +from dataclasses import dataclass #PRODUCTION VALUES @@ -173,11 +174,23 @@ def get_datalist(data, listname): return data[listname] if data else [] +class LDAPSearch: + """ Wrapper class for LDAP searches. """ + server: Server = None + connection: Connection = None + + def __init__(self, ldap_server, ldap_user, ldap_authtok): + self.server = Server(ldap_server, get_info=ALL) + self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + + def search(self, ou, filter_str, attrs): + _, _, response, _ = self.connection.search(f"ou={ou},o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=attrs) + return response + def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): ldap_group_osggids = set() - server = Server(ldap_server, get_info=ALL) - connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=["gidNumber"]) + searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) + response = searcher.search("groups", "(cn=*)", ["gidNumber"]) for group in response: ldap_group_osggids.add(group["attributes"]["gidNumber"]) return ldap_group_osggids @@ -188,9 +201,10 @@ def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filte ldap_active_users = dict() filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") - server = Server(ldap_server, get_info=ALL) - connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=people,o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=["employeeNumber", "isMemberOf"]) + + searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok) + response = searcher.search("people", filter_str, ["employeeNumber", "isMemberOf"]) + for person in response: ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", []) From f048953e93b83aabeeff20b1a4edd5831be68392 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Mon, 1 Dec 2025 08:52:56 -0600 Subject: [PATCH 18/18] Extract ldap base dn to variable --- comanage_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/comanage_utils.py b/comanage_utils.py index 0e9f4f0..7f5b5b9 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -17,6 +17,7 @@ PRODUCTION_OSG_CO_ID = 7 PRODUCTION_UNIX_CLUSTER_ID = 1 PRODUCTION_LDAP_TARGET_ID = 6 +LDAP_BASE_DN = "o=OSG,o=CO,dc=cilogon,dc=org" #TEST VALUES @@ -184,7 +185,7 @@ def __init__(self, ldap_server, ldap_user, ldap_authtok): self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) def search(self, ou, filter_str, attrs): - _, _, response, _ = self.connection.search(f"ou={ou},o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=attrs) + _, _, response, _ = self.connection.search(f"ou={ou},{LDAP_BASE_DN}", filter_str, attributes=attrs) return response def get_ldap_groups(ldap_server, ldap_user, ldap_authtok):