Skip to content
Snippets Groups Projects
Commit ea350cbc authored by Miguel Angel Reina Ortega's avatar Miguel Angel Reina Ortega
Browse files

Several fixes:

    - Get the indexes of the change
    - Detect the changed clause
    - Correct generation of the MD with changes marks

Some fixes:
    - the complete ToC clause is analysed (between clause marks)
    - figura caption regexp to catch figure caption in any part of the line
parent 094b1614
No related branches found
No related tags found
No related merge requests found
Pipeline #477 passed
......@@ -63,6 +63,7 @@ Word CR text:
Protect branch:
stage: generation
when: on_success
needs: ["Word CR text"]
only:
- merge_requests
script:
......
......@@ -90,8 +90,17 @@ def find_all_clauses(progress:Progress, mdLines:list[str]):
index = 1
empty = ""
clause = Clause(empty,0,0,empty)
clause = Clause(empty,0,0,"0")
#for line in mdLines:
# if line.startswith('#'):
# # Clause 0 (from start to first clause) found
# clause.to_id = index - 1
# clauses.append(clause)
# break
# index = index + 1
index = 1
for line in mdLines:
if line.startswith('#'):
matches = re.findall(clauseregex, line) # Match heading
......@@ -113,7 +122,12 @@ def find_all_clauses(progress:Progress, mdLines:list[str]):
clause.to_id = index - 1
clauses.append(clause)
logging.debug("Number of clauses: {len(clauses)}")
logging.debug(f"Number of clauses: {len(clauses)}")
for clause in clauses:
logging.debug(clause.clause_nr)
logging.debug(clause.from_id)
logging.debug(clause.to_id)
return clauses
class MR:
......@@ -155,25 +169,28 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st
logging.debug(f"Looking at changes in {patched_file.source_file}")
lines_added = 0
lines_removed = 0
previous_change_lines_added = 0
for change in patched_file:
logging.debug(f'Change from patch details: source_start: {change.source_start} - target_start: {change.target_start}')
change_start_line, change_end_line, change_lines_added, change_lines_removed = changeDetails(change)
logging.debug(f"Change starting at line {change_start_line} and ending at line {change_end_line}")
logging.debug(f"Change containing {change_lines_added} added lines and {change_lines_removed} lines removed")
lines_added = lines_added + change_lines_added
lines_removed = lines_removed + change_lines_removed
# Check the previous changed_clause
if (changed_clause.from_id <= change_start_line) and (changed_clause.to_id + lines_added - lines_removed >= change_end_line):
generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, True)
if (changed_clause.from_id <= change_start_line) and (changed_clause.to_id >= change_end_line):
generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, True, previous_change_lines_added)
continue
i = 0
# Check all clauses
for clause in clauses:
if (clause.from_id <= change_start_line) and (clause.to_id + lines_added - lines_removed >= change_end_line):
if (clause.from_id <= change_start_line) and (clause.to_id >= change_end_line):
changed_clause = clauses.pop(i)
changed_clauses.append(clause)
generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, False)
generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, False, previous_change_lines_added)
break
i = i + 1
previous_change_lines_added = change_lines_added
logging.info(f"Total number of lines added is {lines_added}")
logging.info(f"Total number of lines removed is {lines_removed}")
......@@ -186,10 +203,17 @@ def changeDetails(change) -> (int, int, int, int):
i = 0
lines_added = 0
lines_removed = 0
change_start_line = change.target_start
change_start_line = change.source_start
change_end_line = change_start_line
for line in change:
#print(vars(line))
if line.is_added:
logging.debug(f'Line of change: (ADDED) source_line_no: {line.source_line_no} - target_line_no: {line.target_line_no} - value: {line.value}')
elif line.is_removed:
logging.debug(
f'Line of change: (REMOVED) source_line_no: {line.source_line_no} - target_line_no: {line.target_line_no} - value: {line.value}')
else:
logging.debug(f'Line of change: source_line_no: {line.source_line_no} - target_line_no: {line.target_line_no} - value: {line.value}')
if line.is_added or line.is_removed:
#if change_start_line == change.target_start:
# change_start_line = change.target_start + i
......@@ -201,16 +225,21 @@ def changeDetails(change) -> (int, int, int, int):
lines_added = lines_added + 1
elif line.is_removed:
lines_removed = lines_removed + 1
if change_start_line == change.target_start:
if change_start_line == change.source_start:
if line.is_added:
change_start_line = line.target_line_no
change_end_line = change_start_line
if change.source_start > change.target_start: # there are more lines removed than lines added
change_start_line = line.target_line_no - 1 + change.source_start - change.target_start
else:
change_start_line = line.target_line_no - 1 + change.target_start - change.source_start # There are more lines added than removed lines
elif line.is_removed:
change_start_line = line.source_line_no
change_start_line = line.source_line_no - 1
change_end_line = change_start_line
else:
if line.is_added:
change_end_line = line.target_line_no
if change.source_start > change.target_start:
change_end_line = line.target_line_no + change.source_start - change.target_start + lines_removed - lines_added # There are more lines added than removed lines
else:
change_end_line = line.target_line_no + change.target_start - change.source_start + lines_removed - lines_added
elif line.is_removed:
change_end_line = line.source_line_no
#i = i + 1
......@@ -218,7 +247,7 @@ def changeDetails(change) -> (int, int, int, int):
#change_end_line = change_end_line - lines_removed
return change_start_line, change_end_line, lines_added, lines_removed
def generateMDforChange(progress:Progress, mdLines:list[str],changed_clause:Clause, change, outDirectory:str, existing_clause:bool):
def generateMDforChange(progress:Progress, mdLines:list[str],changed_clause:Clause, change, outDirectory:str, existing_clause:bool, previous_change_lines_added:int):
'''
Generate the MD for the clauses that have been modified by the merge request
https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.stfubuntu Returns a list of Clauses, start index and end index
......@@ -235,17 +264,25 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st
else:
clauseMDlines = readMDFile(progress, outDirectory + '/' + changed_clause.clause_nr +'.md')
logging.info(f"Lines in clause {changed_clause.clause_nr} is {len(clauseMDlines)}")
if change.source_start >= changed_clause.from_id:
j = change.source_start - changed_clause.from_id
index_gap = change.target_start - change.source_start
elif change.source_start < changed_clause.from_id:
j = changed_clause.from_id - change.source_start
index_gap = change.source_start - change.target_start
j = change.target_start - changed_clause.from_id + index_gap
if existing_clause:
j = j + previous_change_lines_added
logging.info(f"Change.source_start {change.source_start}")
logging.info(f"Changed_clause.from_id {changed_clause.from_id}")
for line in change:
if line.source_line_no != None and line.source_line_no < changed_clause.from_id:
j = 0
continue
else:
# It should deal with the case where source line number is not within the changed clause index (too many removed lines in other clauses, i.e.)
#if line.source_line_no != None and line.source_line_no < changed_clause.from_id:
# j = j
# continue
#else:
#if (not (line.value.strip() == '') and (line.is_added)):
#print(vars(line))
if line.is_added:
......@@ -257,16 +294,22 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st
modifiedElements.append("<span class=\"underline\">" + element.strip() + "</span> ")
#modifiedRow = "|" + "|".join(modifiedElements) + "|" + "\n"
else:
modifiedElements.append(element)
modifiedRow = "|".join(modifiedElements) + "\n"
clauseMDlines.insert(j,modifiedRow)
modifiedElements.append(" ")
modifiedRow = "|".join(modifiedElements)
clauseMDlines.insert(j,modifiedRow + "\n")
clauseMDlines.pop(j + 1) # Todo Check what happens when modifying last row of a table
else:
if not line.value.strip() == '':
clauseMDlines.insert(j, "<span class=\"underline\">" + line.value.strip() + "</span>\n\n")
if line.value.startswith("!["): # It is a figure
clauseMDlines.insert(j, line.value.strip() + "\n\n")
# clauseMDlines.insert(j, "<span class=\"underline\">" + line.value.strip() + "</span>\n") #Track change OK Caption Not OK
else:
clauseMDlines.insert(j, "<span class=\"underline\">" + line.value.strip() + "</span>\n\n") #it works for simple lines, not for lines in a list
#clauseMDlines.insert(j, "<mark>" + line.value.strip("\n") + "</mark>\n\n")
#if (j + 1) <= len(clauseMDlines):
# clauseMDlines.pop(j+1)
else:
clauseMDlines.insert(j, "\n") #Add an extra line not marked as added
#elif (not (line.value.strip() == '') and (line.is_removed)):
elif line.is_removed:
if line.value.strip().startswith("|"): # It is a table
......@@ -277,13 +320,21 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st
modifiedElements.append("~~" + element.strip() + "~~ ")
#modifiedRow = "|" + "|".join(modifiedElements) + "|" + "\n"
else:
modifiedElements.append(element)
modifiedRow = "|".join(modifiedElements) + "\n"
clauseMDlines.insert(j, modifiedRow)
modifiedElements.append(" ")
modifiedRow = "|".join(modifiedElements)
clauseMDlines.insert(j, modifiedRow + "\n")
if (j + 1) <= len(clauseMDlines):
clauseMDlines.pop(j+1)
else:
logging.debug(f"Index to delete is {j}")
if not line.value.strip() == '':
clauseMDlines.insert(j, "~~" + line.value.strip() + "~~\n")
clauseMDlines.insert(j, "~~" + line.value.strip() + "~~\n\n")
#else:
# clauseMDlines.insert(j, "~~\t~~\n")
#clauseMDlines.insert(j, "\n")
if (j + 1) <= len(clauseMDlines):
logging.debug(f'Line marked as removed: {clauseMDlines[j]}')
logging.debug(f'Line being removed: {clauseMDlines[j+1]}')
clauseMDlines.pop(j+1)
j = j + 1
......@@ -310,6 +361,7 @@ def main(args=None):
parser.add_argument('mergeID', help="Merge IID")
pargs = parser.parse_args()
logging.basicConfig(level=logging.INFO)
# Process documents and print output
......
......@@ -67,6 +67,8 @@ def correctTOC(progress:Progress, mdLines:list[str], tocSection:str = 'Contents'
continue
else:
_lines.append(line)
else:
_lines.append(line)
progress.stop_task(_taskID)
return _lines
......@@ -112,11 +114,10 @@ def replaceFigureCaptions(progress:Progress, mdLines:list[str]) -> list[str]:
if matches:
# Replace the previous figure markdown name with the captions
_idx = len(_lines) - 1
while _idx >= 0 and not _lines[_idx].startswith('!['):
while _idx >= 0 and _lines[_idx].count('![') == 0:
_idx -= 1
if _idx > 0:
_lines[_idx] = re.sub(r'^.*?]', f'![{matches[0]}]', _lines[_idx])
_lines[_idx] = re.sub(r'!.*]', f'![{matches[0]}]', _lines[_idx])
elif re.findall(nocaptionfigureregex, line):
_lines.append(re.sub(r'^.*?]', f'![]', line))
else:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment