Fixed a critical bug in the recommendation scores

Author: Jim Avery

PostgreSQL/initmovielens1mdatabase.sql

@@ -14,9 +14,9 @@ create index itemrating_index on ratings(itemid);
 
 set client_encoding = LATIN1;
 
-COPY users(userid,gender,age,job,zipcode) from '/home/administrator/javery/recdb-postgresql/PostgreSQL/moviedata/MovieLens1M/users.dat' DELIMITERS ';';
-COPY items(itemid,name,genre) from '/home/administrator/javery/recdb-postgresql/PostgreSQL/moviedata/MovieLens1M/movies.dat' DELIMITERS ';';
-COPY ratings(userid,itemid,ratingval,ratingts) from '/home/administrator/javery/recdb-postgresql/PostgreSQL/moviedata/MovieLens1M/ratings.dat' DELIMITERS ';';
+COPY users(userid,gender,age,job,zipcode) from '/home/jim/RecDBnew/recdb-postgresql/PostgreSQL/moviedata/MovieLens1M/users.dat' DELIMITERS ';';
+COPY items(itemid,name,genre) from '/home/jim/RecDBnew/recdb-postgresql/PostgreSQL/moviedata/MovieLens1M/movies.dat' DELIMITERS ';';
+COPY ratings(userid,itemid,ratingval,ratingts) from '/home/jim/RecDBnew/recdb-postgresql/PostgreSQL/moviedata/MovieLens1M/ratings.dat' DELIMITERS ';';
 
 CREATE TABLE recathonheavyusers (userid integer primary key);
 insert into recathonheavyusers select t.userid from (select userid,count(*) as rating from ratings group by userid order by rating desc) t limit 100;

PostgreSQL/src/backend/executor/execRecommend.c

@@ -477,12 +477,13 @@ ExecFilterRecommend(RecScanState *recnode,
 			}
 
 			/* If we get here, then we found a user who will be actually
-			 * returned in the results. */
+			 * returned in the results. One quick reset here. */
+			recnode->fullItemNum = 0;
 		}
 
 		/* Mark the user ID and index. */
 		attributes->userID = userID;
-		recnode->userindex = recnode->userNum;
+		recnode->userindex = userindex;
 
 		/* With the user ID determined, we need to investigate and see
 		 * if this is a new user. If so, attempt to create prediction
@@ -495,9 +496,13 @@ ExecFilterRecommend(RecScanState *recnode,
 
 		/* Now replace the item ID, if the user is valid. Otherwise,
 		 * leave the item ID as is, as it doesn't matter what it is. */
-		itemindex = recnode->itemNum;
 		if (recnode->validUser)
-			itemID = recnode->itemList[itemindex];
+			itemID = recnode->itemList[recnode->itemNum];
+		while (recnode->fullItemList[recnode->fullItemNum] < itemID)
+			recnode->fullItemNum++;
+		itemindex = recnode->fullItemNum;
+		if (recnode->fullItemList[itemindex] > itemID)
+			elog(ERROR, "critical item mismatch in ExecRecommend");
 
 		/* Plug in the data, marking those columns full. We also need to
 		 * mark the rating column with something temporary. */
@@ -524,6 +529,7 @@ ExecFilterRecommend(RecScanState *recnode,
 			recnode->userNum++;
 			recnode->newUser = true;
 			recnode->itemNum = 0;
+			recnode->fullItemNum = 0;
 			if (recnode->userNum >= recnode->totalUsers)
 				recnode->finished = true;
 		}
@@ -974,10 +980,6 @@ ExecRecScan(RecScanState *node)
 void
 ExecEndRecScan(RecScanState *node)
 {
-	AttributeInfo* attributes;
-
-	attributes = (AttributeInfo*) node->attributes;
-
 	/* End the normal scan. */
 	switch(nodeTag(node->subscan)) {
 		case T_SeqScanState:
@@ -992,6 +994,8 @@ ExecEndRecScan(RecScanState *node)
 	/* Now for extra stuff. */
 	if (node->itemList)
 		pfree(node->itemList);
+	if (node->fullItemList)
+		pfree(node->fullItemList);
 	if (node->userFeatures)
 		pfree(node->userFeatures);
 	if (node->base_slot)

PostgreSQL/src/backend/executor/nodeRecjoin.c

@@ -12,6 +12,8 @@
  * IDENTIFICATION
  *	  src/backend/executor/nodeRecjoin.c
  *
+ * THIS CODE IS CURRENTLY NOT INCORPORATED IN RECDB.
+ *
  *-------------------------------------------------------------------------
  */
 /*
@@ -114,7 +116,7 @@ ExecRecJoin(RecJoinState *recjoin)
 	 * until we're done projecting out tuples from a join tuple.
 	 */
 	ResetExprContext(econtext);
-printf("RecJoin.\n");
+
 	/*
 	 * Ok, everything is setup for the join. We're going to get exactly one
 	 * tuple from the outer plan, because we just want to use its tupleDesc
@@ -131,23 +133,21 @@ printf("RecJoin.\n");
 		int i, userID, innerItemID, natts;
 		GenRating *findRating;
 		bool minimalTuple = false;
-printf("Point 1.\n");
+
 		/*
 		 * If we need an outer tuple, we fetch one. This creates a few
 		 * structures that we need to effectively perform a RecJoin.
 		 * It works both for initializing and resuming the inner loop.
 		 */
 		if (recjoin->rj_NeedNewOuter) {
 			GenRating *tempItem;
-printf("Point 2.\n");
 
 			outerTupleSlot = ExecProcNode(outerPlan);
 			/* If this happens, we're out of users. */
 			if (TupIsNull(outerTupleSlot)) {
 				ENL1_printf("no outer tuple (out of users), ending join");
 				return NULL;
 			}
-printf("Point 3.\n");
 
 			/* Otherwise, we need to construct our hash table, since
 			 * we need info from the previous operator to do so. */
@@ -163,20 +163,17 @@ printf("Point 3.\n");
 
 				hashAdd(recjoin->itemTable,tempItem);
 			}
-printf("Point 4.\n");
 
 			/* Then we'll do some other stuff to ensure the loop
 			 * runs correctly. */
 			recjoin->rj_NeedNewOuter = false;
 			ENL1_printf("rescanning inner plan");
 			ExecReScan(innerPlan);
 		}
-printf("Point 5.\n");
 
 		/* We construct a new tuple on the fly. */
 		outerTupleSlot = MakeSingleTupleTableSlot(recnode->base_slot);
 		outerTupleSlot->tts_isempty = false;
-printf("Point 6.\n");
 
 		/* Mark all slots as non-empty and zero. */
 		natts = outerTupleSlot->tts_tupleDescriptor->natts;
@@ -186,7 +183,6 @@ printf("Point 6.\n");
 			outerTupleSlot->tts_isnull[i] = false;
 			outerTupleSlot->tts_nvalid++;
 		}
-printf("Point 7.\n");
 
 		/*
 		 * try to get the next inner tuple.
@@ -202,17 +198,16 @@ printf("Point 7.\n");
 		if (recjoin->innerTupleAtt < 0) {
 			for (i = 0; i < innerTupleSlot->tts_tupleDescriptor->natts; i++) {
 				char* col_name = innerTupleSlot->tts_tupleDescriptor->attrs[i]->attname.data;
-//printf("%s\n",col_name);
+
 				if (strcmp(col_name,attributes->itemkey) == 0) {
 					recjoin->innerTupleAtt = i;
 					break;
 				}
 			}
 		}
-printf("Point 7.5.\n");
+
 		if (!TupIsNull(innerTupleSlot) && innerTupleSlot->tts_mintuple)
 			minimalTuple = true;
-printf("Point 8.\n");
 
 		/* If there's no inner tuple, then we'll make a note to reset the
 		 * inner loop and get a new outer tuple. */
@@ -222,7 +217,6 @@ printf("Point 8.\n");
 			recjoin->rj_NeedNewOuter = true;
 			continue;
 		}
-printf("Point 9.\n");
 
 		/*
 		 * We now have an inner tuple and a shell of an outer tuple. We need
@@ -247,18 +241,12 @@ printf("Point 9.\n");
 			FreeTupleDesc(tempdesc);*/
 		}
 		userID = attributes->userID;
-printf("Point 10.\n");
 
 		/*
 		 * Is this item ID one of the ones we need to predict a rating for?
 		 */
-if (!recjoin->itemTable) printf("No table.\n");
-printf("innerItemID = %d\n",innerItemID);
-printf("userID = %d\n",userID);
-if (innerItemID < 0) {printf("Skipping item.\n");continue;}
 		findRating = hashFind(recjoin->itemTable,innerItemID);
 		if (!findRating) continue;
-printf("Point 11.\n");
 
 		/*
 		 * We're ok to construct a tuple at this point.
@@ -269,7 +257,6 @@ printf("Point 11.\n");
 		outerTupleSlot->tts_isnull[recnode->itematt] = false;
 
 		econtext->ecxt_outertuple = outerTupleSlot;
-printf("Point 12.\n");
 
 		/*
 		 * at this point we have a new pair of inner and outer tuples so we
@@ -284,23 +271,20 @@ printf("Point 12.\n");
 		if (ExecQual(joinqual, econtext, false))
 		{
 			node->nl_MatchedOuter = true;
-printf("Point 13.\n");
 
 			/* In an antijoin, we never return a matched tuple */
 			if (node->js.jointype == JOIN_ANTI)
 			{
 				node->nl_NeedNewOuter = true;
 				continue;		/* return to top of loop */
 			}
-printf("Point 14.\n");
 
 			/*
 			 * In a semijoin, we'll consider returning the first match, but
 			 * after that we're done with this outer tuple.
 			 */
 			if (node->js.jointype == JOIN_SEMI)
 				node->nl_NeedNewOuter = true;
-printf("Point 15.\n");
 
 			if (otherqual == NIL || ExecQual(otherqual, econtext, false))
 			{
@@ -310,7 +294,6 @@ printf("Point 15.\n");
 				 */
 				TupleTableSlot *result;
 				ExprDoneCond isDone;
-printf("Point 16.\n");
 
 				/*
 				 * The tuples match our qualifications. We now apply
@@ -319,12 +302,10 @@ printf("Point 16.\n");
  				 */
 				int itemindex = binarySearch(recnode->fullItemList, innerItemID, 0, recnode->fullTotalItems);
 				applyRecScore(recnode, outerTupleSlot, innerItemID, itemindex);
-printf("Point 17.\n");
 
 				ENL1_printf("qualification succeeded, projecting tuple");
 
 				result = ExecProject(node->js.ps.ps_ProjInfo, &isDone);
-printf("Point 18.\n");
 
 				if (isDone != ExprEndResult)
 				{
@@ -343,7 +324,6 @@ printf("Point 18.\n");
 		 * Tuple fails qual, so free per-tuple memory and try again.
 		 */
 		ResetExprContext(econtext);
-printf("Point 19.\n");
 
 		ENL1_printf("qualification failed, looping");
 	}
@@ -356,8 +336,6 @@ printf("Point 19.\n");
 RecJoinState *
 ExecInitRecJoin(RecJoin *node, EState *estate, int eflags)
 {
-	int i;
-	GenRating *tempItem;
 	RecJoinState *rjstate;
 
 	rjstate = makeNode(RecJoinState);

PostgreSQL/src/backend/parser/parse_rec.c

@@ -35,8 +35,8 @@ static void modifyColumnRef(ColumnRef *attribute, char *recname, char *viewname)
 static void modifyFrom(SelectStmt *stmt, RecommendInfo *recInfo);
 static void filterfirst(Node *whereExpr, RecommendInfo *recInfo);
 static bool filterfirstrecurse(Node *whereExpr, RecommendInfo *recInfo);
-static void applyRecJoin(Node *whereClause, List *fromClause, RecommendInfo *recInfo);
-static RangeVar* locateJoinTable(Node* recExpr, List *fromClause, RangeVar* eventtable, char* key);
+//static void applyRecJoin(Node *whereClause, List *fromClause, RecommendInfo *recInfo);
+//static RangeVar* locateJoinTable(Node* recExpr, List *fromClause, RangeVar* eventtable, char* key);
 static bool tableMatch(RangeVar* table, char* tablename);
 
 /*
@@ -687,8 +687,9 @@ filterfirstrecurse(Node *whereExpr, RecommendInfo *recInfo) {
 /*
  * applyRecJoin -
  *	  A function to determine if we need to employ a RecJoin.
+ *	  CURRENTLY NOT IN USE.
  */
-static void
+/*static void
 applyRecJoin(Node *whereClause, List *fromClause, RecommendInfo *recInfo) {
 	RangeVar *partnerTable;
 	AttributeInfo *attributes = recInfo->attributes;
@@ -705,19 +706,10 @@ applyRecJoin(Node *whereClause, List *fromClause, RecommendInfo *recInfo) {
 	if (!partnerTable)
 		partnerTable = locateJoinTable(whereClause, fromClause,
 					recInfo->recommender, attributes->userkey);
-//	else
-//		ereport(ERROR,
-//		(errcode(ERRCODE_SYNTAX_ERROR),
-//		 errmsg("found item ID match")));
-
 
 	// If we found no such table, give up.
 	if (!partnerTable)
 		return;
-//	else
-//		ereport(ERROR,
-//		(errcode(ERRCODE_SYNTAX_ERROR),
-//		 errmsg("found user ID match")));
 
 	// Otherwise, we found an appropriate table. Make a note.
 	recInfo->opType = OP_JOIN;
@@ -726,23 +718,24 @@ applyRecJoin(Node *whereClause, List *fromClause, RecommendInfo *recInfo) {
 	partnerInfo = makeNode(RecommendInfo);
 	partnerInfo->opType = OP_JOINPARTNER;
 	partnerTable->recommender = partnerInfo;
-}
+}*/
 
 
 /*
  * locateJoinTable -
  *	  A function to search through the WHERE clause and see if we are
  *	  joining our recommender with some other table, either by item ID
  *	  or user ID. We give preference to item ID.
+ *	  CURRENTLY NOT IN USE.
  */
-static RangeVar*
+/*static RangeVar*
 locateJoinTable(Node* recExpr, List *fromClause, RangeVar* eventtable, char* key) {
 	A_Expr *recAExpr;
 
 	if (!recExpr)
 		return NULL;
 
-	/* Turns out this isn't necessarily an A_Expr. */
+	// Turns out this isn't necessarily an A_Expr.
 	if (nodeTag(recExpr) != T_A_Expr)
 		return NULL;
 
@@ -835,7 +828,7 @@ locateJoinTable(Node* recExpr, List *fromClause, RangeVar* eventtable, char* key
 
 	// All other kinds fail, at least for now.
 	return NULL;
-}
+}*/
 
 /*
  * tableMatch -

PostgreSQL/src/backend/utils/misc/recathon.c

@@ -2961,7 +2961,7 @@ generateItemCosModel(RecScanState *recnode) {
 	PlanState *simplanstate;
 	TupleTableSlot *simslot;
 	MemoryContext simcontext;
-printf("Starting model.\n");
+
 	attributes = (AttributeInfo*) recnode->attributes;
 	eventtable = attributes->eventtable;
 	userkey = attributes->userkey;
@@ -3026,7 +3026,7 @@ printf("Starting model.\n");
 
 	/* Query cleanup. */
 	recathon_queryEnd(simqueryDesc, simcontext);
-printf("Calculating similarities.\n");
+
 	/* Now we do the similarity calculations. Note that we
 	 * don't include duplicate entries, to save time and space.
 	 * The first item ALWAYS has a lower value than the second. */
@@ -3069,8 +3069,6 @@ printf("Calculating similarities.\n");
 	recnode->fullTotalItems = numItems;
 	recnode->fullItemList = itemIDs;
 	recnode->itemCFmodel = itemmodel;
-
-printf("Model complete.\n");
 }
 
 /* ----------------------------------------------------------------
@@ -4026,7 +4024,6 @@ prepUserForRating(RecScanState *recstate, int userID) {
 				applyItemSimGenerate(recstate);
 			else
 				applyItemSim(recstate, attributes->recModelName);
-
 			break;
 		case userCosCF:
 		case userPearCF: