Initial Commit for the new project-usermap script

Uses LDAP search to find active and provisioned groups then compares them to the list project groups in COmanage to determine which LDAP groups and users to build the map out of.

Author: williamnswanson

osg-comanage-project-usermap.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
+import re
 import os
 import sys
 import getopt
+import subprocess
 import collections
 import urllib.error
 import urllib.request
@@ -13,6 +15,36 @@
 ENDPOINT = "https://registry-test.cilogon.org/registry/"
 OSG_CO_ID = 8
 
+LDAP_AUTH_COMMAND = [
+    "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf",
+]
+
+LDAP_GROUP_MEMBERS_COMMAND = [
+    "ldapsearch",
+    "-H",
+    "ldaps://ldap.cilogon.org",
+    "-D",
+    "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
+    "-w", "{}",
+    "-b",
+    "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org",
+    "-s",
+    "one",
+    "(cn=*)",
+]
+
+LDAP_ACTIVE_USERS_COMMAND = [
+    "ldapsearch",
+    "-LLL",
+    "-H", "ldaps://ldap.cilogon.org",
+    "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
+    "-x",
+    "-w",  "{}",
+    "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org",
+    "(isMemberOf=CO:members:active)", "voPersonApplicationUID",
+    "|", "grep", "voPersonApplicationUID",
+    "|", "sort",
+]
 
 _usage = f"""\
 usage: [PASS=...] {SCRIPT} [OPTIONS]
@@ -64,13 +96,19 @@ def get_osg_co_groups__map():
     return { g["Id"]: g["Name"] for g in data }
 
 
-def co_group_is_ospool(gid):
+def co_group_is_project(gid):
     #print(f"co_group_is_ospool({gid})")
     resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr)
     data = utils.get_datalist(resp_data, "Identifiers")
     return any( i["Type"] == "ospoolproject" for i in data )
 
 
+def get_co_group_osggid(gid):
+    resp_data = get_co_group_identifiers(gid)
+    data = get_datalist(resp_data, "Identifiers")
+    return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"]
+
+
 def get_co_group_members__pids(gid):
     #print(f"get_co_group_members__pids({gid})")
     resp_data = utils.get_co_group_members(gid,  options.endpoint, options.authstr)
@@ -115,6 +153,84 @@ def parse_options(args):
         usage("PASS required")
 
 
+def get_ldap_group_members_data():
+    gidNumber_str = "gidNumber: "
+    gidNumber_regex = re.compile(gidNumber_str)
+    member_str = f"hasMember: "
+    member_regex = re.compile(member_str)
+
+    auth_str = subprocess.run(
+            LDAP_AUTH_COMMAND,
+            stdout=subprocess.PIPE
+            ).stdout.decode('utf-8').strip()
+    
+    ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND
+    ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str
+
+    data_file = subprocess.run(
+        ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')
+
+    search_results = list(filter( 
+        lambda x: not re.compile("#|dn|cn|objectClass").match(x),
+        (line for line in data_file)))
+        
+    search_results.reverse()
+
+    group_data_dict = dict()
+    index = 0
+    while index < len(search_results) - 1:
+        while not gidNumber_regex.match(search_results[index]):
+            index += 1
+        gid = search_results[index].replace(gidNumber_str, "")
+        members_list = []
+        while search_results[index] != "":
+            if member_regex.match(search_results[index]):
+                members_list.append(search_results[index].replace(member_str, ""))
+            index += 1
+        group_data_dict[gid] = members_list
+        index += 1
+
+    return group_data_dict
+
+
+def get_ldap_active_users():
+    auth_str = subprocess.run(
+            LDAP_AUTH_COMMAND,
+            stdout=subprocess.PIPE
+            ).stdout.decode('utf-8').strip()
+    
+    ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND
+    ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str
+
+    active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')
+    users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users)
+    return users
+
+
+def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names):
+    users_to_projects_map = dict()
+    for osggid in project_to_user_map:
+        for user in project_to_user_map[osggid]:
+            if user in active_users:
+                if user not in users_to_projects_map:
+                    users_to_projects_map[user] = [osggids_to_names[osggid]]
+                else:
+                    users_to_projects_map[user].append(osggids_to_names[osggid])
+
+    return users_to_projects_map
+
+
+def get_co_api_data():
+    #TODO add cacheing for COManage API data
+
+    groups = get_osg_co_groups__map()
+    project_osggids_to_name = dict()
+    for id,name in groups.items():
+        if co_group_is_project(id):
+            project_osggids_to_name[get_co_group_osggid(id)] = name
+    return project_osggids_to_name
+
+
 def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser):
     pid_gids = collections.defaultdict(set)
 
@@ -134,17 +250,29 @@ def filter_by_group(pid_gids, groups, filter_group_name):
 
 
 def get_osguser_groups(filter_group_name=None):
-    groups = get_osg_co_groups__map()
-    ospool_gids = filter(co_group_is_ospool, groups)
-    gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids }
-    all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] )
-    pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids }
-    pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser)
-    if filter_group_name is not None:
-        pid_gids = filter_by_group(pid_gids, groups, filter_group_name)
-
-    return { pid_osguser[pid]: sorted(map(groups.get, gids))
-             for pid, gids in pid_gids.items() }
+    project_osggids_to_name = get_co_api_data()
+    ldap_groups_members = get_ldap_group_members_data()
+    ldap_users = get_ldap_active_users()
+
+    active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys()))
+    project_to_user_map = {
+        osggid : ldap_groups_members[osggid]
+        for osggid in active_project_osggids
+        }
+    all_project_users = set(
+        username for osggid in project_to_user_map for username in project_to_user_map[osggid]
+        )
+    all_active_project_users = all_project_users.intersection(ldap_users)
+    usernames_to_project_map = create_user_to_projects_map(
+                                project_to_user_map,  
+                                all_active_project_users,
+                                project_osggids_to_name,
+                                )
+    
+    #if filter_group_name is not None:
+        #pid_gids = filter_by_group(pid_gids, groups, filter_group_name)
+
+    return usernames_to_project_map
 
 
 def print_usermap_to_file(osguser_groups, file):