import os import sys import numpy as np """ Adapted from the homework assignment from Computer Vision I by Stefan Roth Teaching assistants Shweta Mahajan and Krishnakant Singh From the solution of group 22 (Merve Bektas, Danail Iordanov and Nick Steyer) """ def ransac_iters(p, k, z): """ Computes the required number of iterations for RANSAC. Args: p: probability that any given correspondence is valid k: number of pairs z: total probability of success after all iterations Returns: minimum number of required iterations """ def log(base, argument): """ Computes the logarithm to a given base for a given argument :param base: The base of the logarithm :param argument: The argument of the logarithm :return: The logarithm """ return np.log(argument) / np.log(base) """ Reference: Lecture 8 Slide 23 Probability that we have no outlier in an iteration: p ** k Probability that we have an outlier in an iteration: 1 - p ** k Probability that we have an outlier in each iteration: (1 - p ** k) ** number_of_iterations (1 - p ** k) ** number_of_iterations should be smaller than or equal to 1 - z So number_of_iterations must be at least the logarithm of (1 - z) to the base (1 - p ** k) We always round up to the nearest integer using numpy.ceil because we can only execute full loops. """ return int(np.ceil(log(1 - p ** k, 1 - z))) def ransac(pairs, n_iters, k, threshold): """ RANSAC algorithm. Args: pairs: (l, 4) numpy array containing matched keypoint pairs n_iters: number of ransac iterations threshold: inlier detection threshold Returns: H: (3, 3) numpy array, best homography observed during RANSAC max_inliers: number of inliers N inliers: (N, 4) numpy array containing the coordinates of the inliers :param pairs: :param n_iters: :param threshold: :param k: """ h = {} max_inliers = 0 inliers = {} """ We need to split pairs in two arrays with two columns so that we can pass it to pick_samples. """ p1, p2 = np.hsplit(pairs, 2) """ We execute the ransac loop n_iters times, so that we have a good chance to have a valid homography. """ for iteration in range(n_iters): """ First, we pick a sample of k corresponding point pairs """ sample1, sample2 = pick_samples(p1, p2, k) """ The points in the samples then have to be conditioned, so their values are between -1 and 1. We also retrieve the condition matrices from this function. """ sample1_conditioned, t_1 = condition_points(sample1) sample2_conditioned, t_2 = condition_points(sample2) """ Using the conditioned coordinates, we calculate the homographies both with respect to the conditioned points and the unconditioned points. """ h_current, hc = compute_homography(sample1_conditioned, sample2_conditioned, t_1, t_2) """ We then use the unconditioned points (as clarified in the office hours) to compute the homography distances. """ homography_distances = compute_homography_distance(h_current, p1, p2) """ Using the find_inliers function, we retrieve the inliers for our given homography as well as the number of inliers we currently have. """ number_inliers_current, inliers_current = find_inliers(pairs, homography_distances, threshold) """ If the number of inliers is higher than for a previously computed homography (i.e. our current homography is better), we save our current homography as well as the number of inliers and the inliers themselves. """ if number_inliers_current > max_inliers: h = h_current max_inliers = number_inliers_current inliers = inliers_current report_progress(round((iteration + 1) / n_iters * 100)) """ We then return the best homography we found as well as the number of inliers found with it and the inliers themselves. """ return h, max_inliers, inliers def recompute_homography(inliers): """ Recomputes the homography matrix based on all inliers. Args: inliers: (N, 4) numpy array containing coordinate pairs of the inlier points Returns: H: (3, 3) numpy array, recomputed homography matrix """ """ We split the inliers matrix into two arrays with 2 columns each (one array for the points in the left image and one for the points in the right image). """ p1, p2 = np.hsplit(inliers, 2) """ We then condition these points. """ p1_conditioned, t_1 = condition_points(p1) p2_conditioned, t_2 = condition_points(p2) """ Using the conditioned points, we recompute the homography. The new homography should give better results than the previously computed one, since we only use inliers (correct correspondences) here instead of all correspondences including the wrong ones. """ h, hc = compute_homography(p1_conditioned, p2_conditioned, t_1, t_2) return h def pick_samples(p1, p2, k): """ Randomly select k corresponding point pairs. Args: p1: (n, 2) numpy array, given points in first image p2: (m, 2) numpy array, given points in second image k: number of pairs to select Returns: sample1: (k, 2) numpy array, selected k pairs in left image sample2: (k, 2) numpy array, selected k pairs in right image """ """ We assume that m = n (as confirmed in the office hours) """ """ Reference: https://numpy.org/doc/stable/reference/random/index.html We calculate the number of points n (= m) and generate k unique random integers in [0; n) (random_numbers). We then use the random numbers as indices to select points from p1 and the corresponding points from p2. """ n = p1.shape[0] generator = np.random.default_rng() random_numbers = generator.choice(n, size=k, replace=False) # random_numbers = np.array([96, 93, 63, 118]) sample1 = p1[random_numbers] sample2 = p2[random_numbers] return sample1, sample2 def condition_points(points): """ Conditioning: Normalization of coordinates for numeric stability by subtracting the mean and dividing by half of the component-wise maximum absolute value. Further, turns coordinates into homogeneous coordinates. Args: points: (l, 2) numpy array containing not normalized cartesian coordinates. Returns: ps: (l, 3) numpy array containing normalized points in homogeneous coordinates. T: (3, 3) numpy array, transformation matrix for conditioning """ """ First, we calculate half of the component-wise maximum absolute value for all points, as discussed in the office hours. """ s = np.max(np.abs(points), axis=0) / 2 sx = s[0] sy = s[1] """ We then compute the component-wise mean of all points. """ t = np.mean(points, axis=0) tx = t[0] ty = t[1] """ We then calculate the transformation matrix T like on slide 18 in Lecture 8. We use the x components of s and t in the first row and the y components of s and t in the second row. """ t = np.array(([1 / sx, 0, - tx / sx], [0, 1 / sy, - ty / sy], [0, 0, 1])) """ We now iterate over each point and transform it into homogeneous coordinates. Then we transform the point using T and append the result to the list ps. Finally, the list ps is converted to a numpy array and returned together with the transformation matrix T. """ ps = [] for point in points: homogeneous_point = np.append(point, 1) homogeneous_transformed_point = np.matmul(t, homogeneous_point) ps.append(homogeneous_transformed_point) return np.array(ps), t def compute_homography(p1, p2, t1, t2): """ Estimate homography matrix from point correspondences of conditioned coordinates. Both returned matrices should be normalized so that the bottom right value equals 1. You may use np.linalg.svd for this function. Args: p1: (l, 3) numpy array, the conditioned homogeneous coordinates of interest points in img1 p2: (l, 3) numpy array, the conditioned homogeneous coordinates of interest points in img2 t1: (3,3) numpy array, conditioning matrix for p1 t2: (3,3) numpy array, conditioning matrix for p2 Returns: H: (3, 3) numpy array, homography matrix with respect to unconditioned coordinates HC: (3, 3) numpy array, homography matrix with respect to the conditioned coordinates """ """ Reference: Lecture 8 Slide 9, 13 ff. We did something similar in Assignment 1 """ """ We calculate the number of rows in p1 (which must be the same as number of rows in p2). """ n = p1.shape[0] # Create a (2*N, 9) matrix filled with zeros a = np.zeros((2 * n, 9)) # We iterate from 0 to N (N is the number of correspondences) for i in range(0, n): # The point from the first image (divided by the last component so that x and y have the non-homogeneous # value. We could cut off the last component (which is now 1), but that step is not necessary here.) point1 = p1[i] / p1[i][2] # The corresponding point from the second image (also divided by the last component to get the # non-homogeneous values for x and y) point2 = p2[i] / p2[i][2] # We always calculate two rows of the matrix A in each loop iteration # Here, we calculate the first row according to the lecture slides Lecture 8 slide 14 first_row = np.concatenate((np.zeros(3), point1, - point2[1] * point1)) # Calculate the second row as in Lecture 8 slide 14 second_row = np.concatenate((- point1, np.zeros(3), point2[0] * point1)) # Calculate the row indices at matrix A, where to store the two calculated rows # When we iterate from 0 to N, we get indices 0, 2, 4, 6... etc. for the first row index first_row_index = i * 2 # The second row index is simply the first row index plus one, so the rows are below each other second_row_index = first_row_index + 1 """ Finally, we save the values in the corresponding rows in matrix A """ a[first_row_index] = first_row a[second_row_index] = second_row """ Now we perform a single value decomposition on A and retrieve the last right singular vector """ u, s, vt = np.linalg.svd(a) h = vt[-1] """ Since we used conditioned points, we retrieve the homography matrix HC regarding the conditioned coordinates by reshaping the last right singular vector . """ hc = np.reshape(h, (3, 3)) """ The matrix is then normalized so that the bottom right value equals 1. """ hc = hc / hc[2, 2] """ We calculate the homography matrix with respect to the unconditioned coordinates by using the formula from lecture 8 on slide 18. """ h = np.matmul(np.linalg.inv(t2), np.matmul(hc, t1)) """ The homography matrix is also normalized. """ h = h / h[2, 2] return h, hc def compute_homography_distance(h, p1, p2): """ Computes the pairwise symmetric homography distance. Args: h: (3, 3) numpy array, homography matrix p1: (l, 2) numpy array, interest points in img1 p2: (l, 2) numpy array, interest points in img2 Returns: dist: (l, ) numpy array containing the distances """ """ Determine the number of points in p1 (should be the same as in p2). """ number_of_points = p1.shape[0] """ Calculate the inverse of the homography H. """ h_inverse = np.linalg.inv(h) """ Create a zero-filled array which we later fill with the distances. """ distances = np.zeros(number_of_points) """ We calculate the transformed points for the given homography. """ p1_transformed = transform_pts(np.array(p1), h) p2_transformed = transform_pts(np.array(p2), h_inverse) """ We calculate the symmetric squared distances using the formula in the assignment sheet for every point pair. """ for index in range(0, number_of_points): x1 = p1[index] x2 = p2[index] x1_transformed = p1_transformed[index] x2_transformed = p2_transformed[index] distances[index] = np.linalg.norm(x1_transformed - x2) ** 2 + np.linalg.norm(x1 - x2_transformed) ** 2 return distances def find_inliers(pairs, dist, threshold): """ Return and count inliers based on the homography distance. Args: pairs: (l, 4) numpy array containing keypoint pairs dist: (l, ) numpy array, homography distances for k points threshold: inlier detection threshold Returns: N: number of inliers inliers: (n, 4) """ inliers_list = [] """ We iterate over each pair and use its index in the array and its value. We then use the index to retrieve the corresponding distance for the pair in dist. If the distance is smaller than or equal to the given threshold, we add the pair to the inliers_list. """ for pair_index, pair_value in enumerate(pairs): if dist[pair_index] <= threshold: inliers_list.append(pair_value) """ We then get the number of inliers as the length of the list. Furthermore, we convert the inliers_list to a numpy array. Both values are then returned. """ n = len(inliers_list) inliers = np.array(inliers_list) return n, inliers def transform_pts(p, h): """ Transform p through the homography matrix H. Args: p: (l, 2) numpy array, interest points h: (3, 3) numpy array, homography matrix Returns: points: (l, 2) numpy array, transformed points """ """ Reference: Lecture 8 Slide 8 """ points_list = [] """ Iterate over every point in p. For each point, we first append a 1 to transform the point into homogeneous coordinates. Then, we perform a matrix multiplication of the point with the homography H, as shown on slide 8 in lecture 8. The transformed point is then converted back into non-homogeneous coordinates by dividing it by the last value and then cutting off the last value (which is then 1 again). The transformed points are returned as a numpy array. """ for point_index, point_value in enumerate(p): homogeneous_p = np.append(point_value, 1) homogeneous_transformed_p = np.matmul(h, homogeneous_p) normalized_homogeneous_transformed_p = homogeneous_transformed_p / homogeneous_transformed_p[2] transformed_p = normalized_homogeneous_transformed_p[0:2] points_list.append(transformed_p) return np.array(points_list) def report_progress(percentage): sys.stdout.write('\r') sys.stdout.write("[%-100s] %d%%" % ('=' * percentage, percentage)) sys.stdout.flush() def main(): threshold = 0.02 # inlier threshold p = 0.35 # probability that any given correspondence is valid z = 0.99 # total probability of success after all iterations if len(sys.argv) == 3: threshold = float(sys.argv[2]) elif len(sys.argv) == 4: threshold = float(sys.argv[2]) p = float(sys.argv[3]) elif len(sys.argv) == 5: threshold = float(sys.argv[2]) p = float(sys.argv[3]) z = float(sys.argv[4]) elif len(sys.argv) != 2: print('Usage: hgcalc [threshold [valid_correspondence_probability [total_probability]]]') return calculate_and_save(sys.argv[1], threshold, p, z) def calculate_and_save(pairs_file_path, threshold, p, z): homography_file_path = '.\\homography.csv' pairs = np.loadtxt(pairs_file_path) k = 4 # number of samples drawn at each iteration n_iters = ransac_iters(p, k, z) print('Input file:', pairs_file_path) output_path = os.path.abspath(homography_file_path) print('Output file:', output_path) print('Threshold: %s, Inlier probability: %s, Success probability: %s' % (threshold, p, z)) print('Calculating %s iterations' % n_iters) h, num_inliers, inliers = ransac(pairs, n_iters, k, threshold) print() print('Percentage of inliers:', "{0:.0%}".format(num_inliers / len(pairs))) # recompute homography matrix based on inliers h = recompute_homography(inliers) np.savetxt(homography_file_path, h) print('Homography saved at', output_path) if __name__ == "__main__": main()