About the operation of files under folders

About the operation of files under folders (yolo-v3 data processing process)

1. Rename the image (file) under the folder to 1.jpg, 2.jpg, 3.jpg...

#coding=utf-8
#Function: All files under the specified folder will be traversed and the file name will be changed
import os

fpath ="E:/ID_card/NewID/Dim-FaceID"#Here is the path to your first folder

def filechanger(path):
    filenames = os.listdir(path)#Put all file names under path path path into list filenames
    n = 1
    for file in filenames:#Loop through each file
        filedomain = os.path.abspath(path)
        #file1 = os.listdir(filedomain)
        #path1 = path + "/" + file
        #print(path1)
        oldname = filedomain + os.sep + file
        #print(oldname)
        newname = filedomain + os.sep + str(n) + ".jpg"
        #print(newname)
        os.rename(oldname, newname)
        n += 1
        
filechanger(fpath)

It is noteworthy that the files obtained by this method are not sequential.

filenames = os.listdir(path)
for file in filenames:

2. Random copy folder under n pictures into the specified folder

import os
import random
import shutil

def copyFile(fileDir):
    #Folders saved after copy ing
    tarDir = "E:/ID_card/TEST/Test-FrontID/"
    
    pathDir = os.listdir(fileDir)
    # Random 200 files
    sample = random.sample(pathDir, 200)
    #print (sample)

    for name in sample:
        #name1 = name.replace('jpg', 'xml')
        #print(name)
        #print(name1)
        #print(fileDir1 + name1)
        #print(fileDir + name)
        #print(tarDir + name)
        shutil.copyfile(fileDir + name, tarDir + name)
        #shutil.copyfile(fileDir1 + name1, tarDir1 + name1)
        #os.remove(fileDir + name)
        #os.remove(fileDir1 + name1)

if __name__ == '__main__':
	#The folder where the original document is stored
	fileDir = "E:/ID_card/TEST/Total-FrontID/"
	
    #fileDir1 = "E:/ID_card/TEST/REC-Front/"
    #tarDir1 = "E:/ID_card/TEST/Test-FrontID1/"
    
    copyFile(fileDir)

shutil.copyfile(A, B) copies A to B. Both A and B are file paths.

3. About Picture Horizontal Flip

#-*- coding: UTF-8 -*- 
 
from PIL import Image
import os


fpath ="C:/Users/92377/Desktop/1/"#Here is the path to your first folder

def filechanger(path):
    filenames = os.listdir(path)#Put all file names under path path path into list filenames
    n = 1
    for file in filenames:#Loop through each file
        filedomain = os.path.abspath(path)	
        file1 = os.listdir(filedomain)
        path1 = path + "/" + file
        im = Image.open(path1)
        out = im.transpose(Image.FLIP_LEFT_RIGHT)
        #oldname = filedomain + os.sep + file
        #print(oldname)
        #newname = filedomain + os.sep + str(n) + ".jpg"
        #print(newname)
        out.save("C:/Users/92377/Desktop/2/" + str(n) + ".jpg")
        #os.rename(oldname, newname)
        n += 1

filechanger(fpath)

transpose(Image.FLIP_LEFT_RIGHT) indicates that the read image is flipped horizontally.

4. Write all file names under the folder into a txt

import os

path = "E:/ID_card/TEST2/Total-image-train"
path1 = "E:/ID_card/TEST2/"

filenames = os.listdir(path)  # Put all file names under path path path into list filenames

fo = open(path1 + "train.txt", "w")

for file in filenames:  # Loop through each file

    fo.write("data/custom/images/" + file + "\n")

fo.close()

Files to be entered into TXT are stored in path, and txt files are stored in path1.

5. XML Processing 1 - Change the labeled revolving box data in Labelimg 2 into square box data (no angle)

from xml.etree import ElementTree as ET
import numpy as np
import math
import os

path = "E:/ID_card/TEST2/FaceID-label"
path1 = "E:/ID_card/TEST2/FaceID-label-REC"
filenames = os.listdir(path)  # Put all file names under path path path into list filenames
for file in filenames:  # Loop through each file
    filedomain = os.path.abspath(path)
    filedomain = os.path.join(filedomain, file)
    tree = ET.parse(filedomain)
    root = tree.getroot()


    # Create a two-dimensional array, test1 is modified, and test is modified before
    test = np.zeros((35, 5), dtype=float)
    test1 = np.zeros((35, 4), dtype=float)
    row = 0
    col = 0
    # print(file)
    for i in root:
        # Label Name and Label Properties of Layer 2 Nodes
        # print(i.tag, i.attrib)
        # Traveling through the third layer of an xml document
        for j in i:
            # print(j.tag, j.text)
            if j.tag == "robndbox":
                # print("This is name !!!!!!!!!!!!!!!!!!!!!!")
                col = 0
                for k in j:
                    # print("The " + str(row) + "Row and the " + str(col) + "Col")
                    # print(k.text)
                    test[row][col] = float(k.text)
                    # print(k.tag, k.text)
                    col = col + 1
                row = row + 1
    # print(test)
    # Here, all the data about angles and coordinates are placed in a two-dimensional array called test.

    for i in range(35):
        m = test[i][4]
        if m > math.pi:
          m = 2 * math.pi - m
        if m > 0.2:
            print(file)
        # angle = (m / math.pi) * 180
        w = test[i][2] * math.cos(m) + test[i][3] * math.sin(m)
        h = test[i][2] * math.sin(m) + test[i][3] * math.cos(m)
        test1[i][0] = test[i][0] - 1 / 2 * w  # XMIN
        test1[i][1] = test[i][1] - 1 / 2 * h  # YMIN
        test1[i][2] = test[i][0] + 1 / 2 * w  # XMAX
        test1[i][3] = test[i][1] + 1 / 2 * h  # YMAX
    print(test1)

    row = 0
    for i in root:
        # Label Name and Label Properties of Layer 2 Nodes
        #print(i.tag, i.attrib)
        # Traveling through the third layer of an xml document
        for j in i:
            #print(j.tag, j.text)
            if j.tag == "robndbox":
                j.tag = "bndbox"
            #print("This is name !!!!!!!!!!!!!!!!!!!!!!")
                #col = 0
                for k in j:
                    #print("The " + str(row) + "Row and the " + str(col) + "Col")
                    #print(k.text)
                    #test[row][col] = float(k.text)
                    #print(k.tag, k.text)
                    if k.tag == "cx":
                        k.tag = "xmin"
                        k.text = str(test1[row][0])
                    elif k.tag == "cy":
                        k.tag = "ymin"
                        k.text = str(test1[row][1])
                    elif k.tag == "w":
                        k.tag = "xmax"
                        k.text = str(test1[row][2])
                    elif k.tag == "h":
                        k.tag = "ymax"
                        k.text = str(test1[row][3])
                    elif k.tag == "angle":
                        j.remove(k)
                    # col = col + 1
                row = row + 1
    # write file
    tree.write(path1 + "/" + file, encoding="utf-8")

This method uses ElementTree method to traverse the structure of binary tree in xml. The method first obtains the data to be used and stores it in the array. After relevant calculation, it puts it in the original position. You can change the tag value and so on.

6. XML Processing 2 - Get the data you want in the box xml and normalize it into the specified txt as required in yolo-v3 code

from xml.etree import ElementTree as ET
import numpy as np
import math
import os

path = "E:/ID_card/TEST/Test-FrontID1"
path1 = "E:/ID_card/TEST/Test-FrontID2/"
filenames = os.listdir(path)  # Put all file names under path path path into list filenames
for file in filenames:  # Loop through each file

    file1 = file.replace('.xml', '')
    fo = open(path1 + file1 + ".txt", "w")

    filedomain = os.path.abspath(path)
    filedomain = os.path.join(filedomain, file)
    tree = ET.parse(filedomain)
    root = tree.getroot()

    # Create a two-dimensional array, test1 is modified, test is modified before
    test = np.zeros((35, 5), dtype=float)
    test1 = np.zeros((35, 5), dtype=float)
    row = 0
    col = 0

    Width = 0
    Height = 0

    for i in root:
        # Label Name and Label Properties of Layer 2 Nodes
        #print(i.tag, i.attrib)
        # Traveling through the third layer of an xml document
        if i.tag == "size":
            for q in i:
                if q.tag == "width":
                    Width = float(q.text)
                elif q.tag == "height":
                    Height = float(q.text)
            print("Width: " + str(Width) + " Height: " + str(Height))
        for j in i:
            if j.text == "Name":
                test[row][0] = 0
            elif j.text == "Sex":
                test[row][0] = 1
            elif j.text == "Birth":
                test[row][0] = 3
            elif j.text == "Nation":
                test[row][0] = 2
            elif j.text == "Address":
                test[row][0] = 4
            elif j.text == "ID":
                test[row][0] = 5
            elif j.text == "Face":
                test[row][0] = 6
            elif j.text == "ExtraInfo":
                test[row][0] = 6
            elif j.text == "English":
                test[row][0] = 6
            elif j.text == "Others":
                test[row][0] = 6

            if j.tag == "bndbox":
                col = 0
                for k in j:
                    col = col + 1
                    test[row][col] = float(k.text)
                row = row + 1
    #print(test)
    # Here, all the data about index and coordinates are placed in the two-dimensional array of test.

    for i in range(35):
        if test[i][0] < 6:
            test1[i][0] = test[i][0]  # index transfer
            test1[i][1] = 1/2 * (test[i][1] + test[i][3]) / Width
            test1[i][2] = 1/2 * (test[i][2] + test[i][4]) / Height
            test1[i][3] = (test[i][3] - test[i][1]) / Width
            test1[i][4] = (test[i][4] - test[i][2]) / Height
    #print(test1)

    for i in range(test1.__len__()):
        if test1[i][1] > 0:
            fo.write(str(int(test1[i][0])) + " " + str(test1[i][1]) + " " + str(test1[i][2]) + " " + str(test1[i][3]) + " " + str(test1[i][4]) + "\n")
    fo.close()

7. XML Processing 3 - Maximize crop images without affecting label tags.

from xml.etree import ElementTree as ET
import numpy as np
import math
from PIL import Image
import os
import cv2

img_path = "C:/Users/92377/Desktop/TEST1"
label_path = "C:/Users/92377/Desktop/TEST2"
img_path1 = "C:/Users/92377/Desktop/TEST1-2"
label_path1 = "C:/Users/92377/Desktop/TEST2-2"

filenames = os.listdir(label_path)  # Put all file names under path path path into list filenames
for file in filenames:  # Loop through each file
    filedomain = os.path.abspath(label_path)
    filedomain = os.path.join(filedomain, file)

    tree = ET.parse(filedomain)
    root = tree.getroot()


    # Create a two-dimensional array, test1 is modified, and test is modified before
    test = np.zeros((35, 6), dtype=float)
    test1 = np.zeros((35, 5), dtype=float)
    Crop_size = np.zeros((1, 5), dtype=float)
    row = 0
    col = 0
    print(file)
    for i in root:
        # Label Name and Label Properties of Layer 2 Nodes
        # print(i.tag, i.attrib)
        # Traveling through the third layer of an xml document

        if i.tag == "size":
            for q in i:
                if q.tag == "width":
                    Width = float(q.text)
                elif q.tag == "height":
                    Height = float(q.text)
            print("Width: " + str(Width) + " Height: " + str(Height))

        for j in i:
            # print(j.tag, j.text)
            if j.tag == "name":
                #print(j.text)
                if j.text == "Card":
                    Crop = 1
                else:
                    Crop = 0

            if j.tag == "robndbox":
                # print("This is name !!!!!!!!!!!!!!!!!!!!!!")
                col = 1
                for k in j:
                    # print("The " + str(row) + "Row and the " + str(col) + "Col")
                    # print(k.text)
                    test[row][0] = Crop
                    test[row][col] = float(k.text)
                    # print(k.tag, k.text)
                    col = col + 1
                row = row + 1
    #print("test")
    #print(test)
    # Here, all the data about angles and coordinates are placed in a two-dimensional array called test.

    for i in range(35):
        m = test[i][5]
        if m > math.pi:
            m = 2 * math.pi - m
        if m > 0.2:
            print(file)
        # angle = (m / math.pi) * 180
        w = test[i][3] * math.cos(m) + test[i][4] * math.sin(m)
        h = test[i][3] * math.sin(m) + test[i][4] * math.cos(m)
        test1[i][0] = test[i][0]
        test1[i][1] = test[i][1] - 1 / 2 * w  # XMIN
        test1[i][2] = test[i][2] - 1 / 2 * h  # YMIN
        test1[i][3] = test[i][1] + 1 / 2 * w  # XMAX
        test1[i][4] = test[i][2] + 1 / 2 * h  # YMAX

        if test1[i][0] == 1:
            #print(i)
            Crop_size = test1[i].copy()
            #print(Crop_size)
            #print(Crop_size is test1[i])

    #print("test1")
    #print(test1)

    # Finish cutting
    img = cv2.imread(img_path + "/" + file.replace("xml","jpg"))
    img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))

    #print(img.size)
    #print("left")
    #print(Crop_size[1])
    #print(Crop_size[2])
    #print(Crop_size[3])
    #print(Crop_size[4])

    cropped = img.crop((Crop_size[1], Crop_size[2], Crop_size[3], Crop_size[4]))  # (left, upper, right, lower)
    cropped.save(img_path1 + "/" + file.replace("xml","jpg"))
    
    left = Crop_size[1]
    upper = Crop_size[2]
    right = Crop_size[3]
    lower = Crop_size[4]

    #print(test1)
    # Coordinates after Crop
    for i in range(len(test1)):
        #print(i)
        
        if test1[i][1] != 0:
            
            #print(type(Crop_size), Crop_size is test1)
            test1[i][1] = test1[i][1] - left  # XMIN
            #print(left)
            test1[i][2] = test1[i][2] - upper  # YMIN
            #print(Crop_size[2])
            test1[i][3] = test1[i][3] - left  # XMAX
            #print(Crop_size[1])
            test1[i][4] = test1[i][4] - upper  # YMAX
            #print(Crop_size[2])
    #print(test1)

    New_Wid = right - left
    New_Hei = lower - upper

    row = 0

    for i in root:
        # Label Name and Label Properties of Layer 2 Nodes
        # print(i.tag, i.attrib)
        # Traveling through the third layer of an xml document
        if i.tag == "size":
            for q in i:
                if q.tag == "width":
                    q.text = str(New_Wid)
                elif q.tag == "height":
                    q.text = str(New_Hei)
    
    for i in root.findall("object"):

        Remove_Flag = 0
        
        for j in i:
            if j.tag == "name":

                if j.text == "Card":
                    Remove_Flag = 1
                    #row = row + 1
                    break
                #print(j.text)

            for value in range(5):
                if test1[row][value] < 0:
                    Remove_Flag = 1
                    #row = row + 1
                    break
            if Remove_Flag == 1:
                break

            if j.tag == "robndbox":
                #print("1")
                j.tag = "bndbox"
            
                #col = 0
                for k in j:
                    
                    if k.tag == "cx":
                        k.tag = "xmin"
                        k.text = str(test1[row][1])
                        #print("2")

                    elif k.tag == "cy":
                        k.tag = "ymin"
                        k.text = str(test1[row][2])
                        #print("3")
                        
                    elif k.tag == "w":
                        k.tag = "xmax"
                        k.text = str(test1[row][3])
                        #print("4")
                       
                    elif k.tag == "h":
                        k.tag = "ymax"
                        k.text = str(test1[row][4])
                        #print("5")
                        
                    elif k.tag == "angle":
                        j.remove(k)
                    # col = col + 1
        row = row + 1

        # break comes out.
        if Remove_Flag == 1:
            #print("remove")
            root.remove(i)
            # i = i - 1

        # print("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
        # print(i.text)

    # write file
    tree.write(label_path1 + "/" + file, encoding="utf-8")

Tags: xml encoding

Posted on Mon, 07 Oct 2019 08:16:34 -0700 by The Bat