import cv2
import numpy as np
import scipy.signal as signal
import scipy.fftpack as fftpack
import os
def rgb2yiq(src):
T = np.array([[0.299, 0.587,0.114 ], [ 0.596, -0.274,-0.322], [ 0.212, -0.523,0.312]])
dst = src.transpose(2,0,1)
dst_re = dst.reshape((3,dst.shape[1]*dst.shape[2]))
dst_re = np.dot(T,dst_re)
dst_re = dst_re.reshape(3,dst.shape[1],dst.shape[2])
dst = dst_re.transpose(1,2,0)
dst[dst<0] = 0
dst[dst>1] = 1
return dst
def yiq2rgb(src):
T = np.array([[1.0, 0.956, 0.620], [1.0, -0.272, -0.647],[1.0, -1.108, 1.705]])
dst = src.transpose(2,0,1)
dst_re = dst.reshape((3,dst.shape[1]*dst.shape[2]))
dst_re = np.dot(T,dst_re)
dst_re = dst_re.reshape(3,dst.shape[1],dst.shape[2])
dst = dst_re.transpose(1,2,0)
dst[dst<0] = 0
dst[dst>1] = 1
dst = np.uint8(dst*255)
return dst
#convert each of the frames to the YIQ color space.
def convertVideo_YIQ(inputframes):
# print(len(inputframes))
dstFrame=[]
for frame in inputframes:
#print(frame)
img = rgb2yiq(frame)
dstFrame.append(img)
dstFrame = np.array(dstFrame)
return dstFrame
#convert each of the frames to the RGB color space.
def convertVideo_BGR(inputframes):
# print(len(inputframes))
dstFrame=[]
for frame in inputframes:
#print(frame)
img = yiq2rgb(frame)
img = cv2.cvtColor(img,cv2.COLOR_RGB2BGR)
dstFrame.append(img)
dstFrame = np.array(dstFrame)
return dstFrame
#Build Gaussian Pyramid
def build_gaussian_pyramid(src,level=3):
s=src.copy()
pyramid=[s]
for i in range(level):
s=cv2.pyrDown(s)
pyramid.append(s)
return pyramid
#Build Laplacian Pyramid
##def build_laplacian_pyramid(src,levels=3):
## gaussianPyramid = build_gaussian_pyramid(src, levels)
## pyramid=[]
## for i in range(levels,0,-1):
## GE=cv2.pyrUp(gaussianPyramid[i])
## L=cv2.subtract(gaussianPyramid[i-1],GE)
## pyramid.append(L)
## return pyramid
def build_laplacian_pyramid(src,levels=3):
curr = src.copy()
lapyramid=[]
for i in range(levels-1):
down = cv2.pyrDown(curr)
up = cv2.pyrUp(down,curr.shape[:2])
Lap = curr - up
Lap[Lap<0]=0
#L[L>1]=1
lapyramid.append(Lap)
curr = down
lapyramid.append(curr)
return lapyramid[::-1]
#load video from file
def load_video(video_filename):
cap=cv2.VideoCapture(video_filename)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width, height = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = int(cap.get(cv2.CAP_PROP_FPS))
frames=np.zeros((frame_count,height,width,3),dtype='float')
x=0
while cap.isOpened():
ret,frame=cap.read()
if ret is True:
frames[x]=frame
x+=1
else:
break
return frames,fps
# apply temporal ideal bandpass filter to gaussian video
def temporal_ideal_filter(tensor,low,high,fps,axis=0):
fft=fftpack.fft(tensor,axis=axis)
frequencies = fftpack.fftfreq(tensor.shape[0], d=1.0 / fps)
bound_low = (np.abs(frequencies - low)).argmin()
bound_high = (np.abs(frequencies - high)).argmin()
fft[:bound_low] = 0
fft[bound_high:-bound_high] = 0
fft[-bound_low:] = 0
iff=fftpack.ifft(fft, axis=axis)
return np.abs(iff)
# build gaussian pyramid for video
def gaussian_video(video_tensor,levels=3):
for i in range(0,video_tensor.shape[0]):
frame=video_tensor[i]
pyr=build_gaussian_pyramid(frame,level=levels)
gaussian_frame=pyr[-1]
if i==0:
vid_data=np.zeros((video_tensor.shape[0],gaussian_frame.shape[0],gaussian_frame.shape[1],3))
vid_data[i]=gaussian_frame
return vid_data
#reconstract video from original video and gaussian video
def reconstract_video(amp_video,origin_video,levels=3):
final_video=np.zeros(origin_video.shape)
for i in range(0,amp_video.shape[0]):
img = amp_video[i]
for x in range(levels):
img=cv2.pyrUp(img)
if levels > 3:
img = cv2.resize(img,origin_video[i].shape[1::-1])
img=img+origin_video[i]
final_video[i]=img
return final_video
#save video to files
def save_video(save_video,outpath,name):
os.makedirs(outpath,exist_ok=True)
fourcc = cv2.VideoWriter_fourcc('M','J','P','G')
[height,width]=save_video[0].shape[0:2]
print(outpath+"/"+name+"_out.avi")
writer = cv2.VideoWriter(outpath+"/"+name+"_out.avi", fourcc, 30, (width, height), 1)
for i in range(0,save_video.shape[0]):
writer.write(cv2.convertScaleAbs(save_video[i]))
writer.release()
#build laplacian pyramid for video
def laplacian_video(video_tensor,levels=3):
tensor_list=[]
for i in range(0,video_tensor.shape[0]):
frame=video_tensor[i]
pyr=build_laplacian_pyramid(frame,levels=levels)
if i==0:
for k in range(levels):
tensor_list.append(np.zeros((video_tensor.shape[0],pyr[k].shape[0],pyr[k].shape[1],3)))
for n in range(levels):
tensor_list[n][i] = pyr[n]
return tensor_list
#butterworth bandpass filter
def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
omega = 0.5 * fs
low = lowcut / omega
high = highcut / omega
b, a = signal.butter(order, [low, high], btype='band')
y = signal.lfilter(b, a, data, axis=0)
return y
#reconstract video from laplacian pyramid
def reconstract_from_tensorlist(filter_tensor_list,levels=3):
final=np.zeros(filter_tensor_list[-1].shape)
for i in range(filter_tensor_list[0].shape[0]):
up = filter_tensor_list[0][i]
for n in range(levels-1):
up=cv2.pyrUp(up)
## up=cv2.pyrUp(up)+filter_tensor_list[n + 1][i]
final[i]=up
return final
#restore laplacian Image
def restoe_LapImage(Lpyramid):
print(len(Lpyramid))
levels=len(Lpyramid)-1
cur = Lpyramid[levels]
for i in range(levels,0,-1):
up = cv2.pyrUp(cur,Lpyramid[i-1].shape[:2])
cur = up + Lpyramid[i-1]
return cur
def loadvideo_getfps(videoName):
frames=[]
# os.makedirs(outpath,exist_ok=True)
cap = cv2.VideoCapture("./data/"+videoName+".mp4")
fps = cap.get(cv2.CAP_PROP_FPS)
while cap.isOpened():
ret,frame=cap.read()
if ret is True:
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
frame = np.double(frame)
cv2.normalize(frame, frame, 1, 0, cv2.NORM_MINMAX)
frames.append(frame)
else:
break
frames = np.array(frames)
return fps,frames
#magnify color
def magnify_color(video_name,low,high,levels=5,amplification=50):
# rgbframes,fps = load_video(video_name)
fps,rgbframes = loadvideo_getfps(video_name)
print("rgb:",rgbframes.shape)
yiqFrames = convertVideo_YIQ(rgbframes)
print("yiq:",yiqFrames.shape)
gau_video=gaussian_video(yiqFrames,levels=levels)
print(gau_video.shape)
filtered_tensor=temporal_ideal_filter(gau_video,low,high,fps)
#
amplified_video= filtered_tensor*amplification
final=reconstract_video(amplified_video,yiqFrames,levels=levels)
rgbfinal=convertVideo_BGR(final)
save_video(rgbfinal,"color",video_name)
#manify motion
def magnify_motion(video_name,low,high,levels=3,amplification=50):
fps,rgbframes = loadvideo_getfps(video_name)
yiqFrames = convertVideo_YIQ(rgbframes)
print("rgb:",rgbframes.shape,"yiq:",yiqFrames.shape)
lap_video_list=laplacian_video(yiqFrames,levels=levels)
filter_tensor_list=[]
for i in range(lev
