From bd076f2f97b8b94713603a1a2320089fc98a3549 Mon Sep 17 00:00:00 2001
From: Kelvin Ly <kelvin.ly1618@gmail.com>
Date: Thu, 11 Jul 2019 23:53:19 -0400
Subject: [PATCH] Autofill schem

---
 pcb/iss-lna/autofill_schem.py | 286 ++++++++++++++++++++++++++++++++++
 pcb/iss-lna/iss-lna.bak       |   4 +-
 pcb/iss-lna/iss-lna.sch       |  20 ++-
 3 files changed, 303 insertions(+), 7 deletions(-)
 create mode 100644 pcb/iss-lna/autofill_schem.py

diff --git a/pcb/iss-lna/autofill_schem.py b/pcb/iss-lna/autofill_schem.py
new file mode 100644
index 0000000..f9802e3
--- /dev/null
+++ b/pcb/iss-lna/autofill_schem.py
@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+# TODO: make sure it also works in python2
+
+# version 0.0.2
+
+# updates:
+# - allow multiple input files, using the later ones only for component footprint
+# and distributor
+# - fix code so it works with Kicad 5
+
+import argparse
+
+DISTRIBUTOR_VALUES = ["\"Mouser\""]
+
+class Component(object):
+  def __init__(self, start):
+    self.designator = None
+    self.value = None
+    self.footprint = None
+    self.footprint_row = None
+    # distributor ids, mapped to (line_num, value, all fields
+    # (to make reconstruction easier)
+    self.distributors = dict()
+    self.cls = None
+    self.last_value = None
+    self.num_fields = 0
+
+    # used mainly for parser debugging
+    self.start = start
+
+class DistributorData(object):
+  def __init__(self, id):
+    self.id = id
+
+def quotesplit(line):
+  parts = line.split(" ")
+  ret = []
+  num_quotes = 0
+  incomplete_part = None
+  for part in parts:
+    num_quotes += len([None for c in part if c == '"'])
+    if num_quotes % 2 == 1:
+      if incomplete_part is None:
+        # just started a quoted section
+        incomplete_part = part
+      else:
+        incomplete_part += " " + part
+    else:
+      if incomplete_part is not None:
+        # ending quote is in this part
+        ret.append(incomplete_part + " " + part)
+        incomplete_part = None
+      else:
+        # normal case; pass through
+        ret.append(part)
+  return ret
+
+def parse_lines(f):
+  lines = []
+  components = []
+  component_start = False
+  component = None
+
+  for i, line in enumerate(f):
+    if component_start:
+      if line.startswith("L"):
+        parts = quotesplit(line)
+        component.designator = parts[2].rstrip()
+        component.cls = component.designator[0]
+      elif line.startswith("F"):
+        parts = quotesplit(line)
+        component.num_fields += 1
+        component.last_value = i
+        if parts[1].isdigit():
+          field_type = int(parts[1])
+          if field_type == 1:
+            component.value = parts[2]
+          elif field_type == 2:
+            component.footprint_row = (i, parts)
+            if len(parts[2]) > 2:
+              component.footprint = parts[2]
+          elif field_type > 3:
+            # TODO: make case insensitive
+            if parts[-1][:-1] in DISTRIBUTOR_VALUES:
+              component.distributors[parts[-1][:-1]] = DistributorData(parts[2])
+      elif line.startswith("$EndComp"):
+        component_start = False
+        # ignore power nodes
+        if component.cls != "#":
+          components.append(component)
+
+    if line.startswith("$Comp"):
+      component_start = True
+      component = Component(i)
+    lines.append(line)
+  return (lines, components)
+
+def infer_components(components):
+  # dictionaries in dictionaries:
+  # distributor = (distributor_type, id) so that non-unique ids can be captured
+  # filled_values[cls][value] = [(footprint, [designators], [distributors])]
+  filled_values = dict()
+
+  # first pass to infer footprints and distributors, second pass to fill in details
+  for c in components:
+    if c.cls is None or len(c.cls) == 0:
+      print("Component at {} incorrectly parsed; no cls set".format(c.start))
+    else:
+      if c.cls not in filled_values:
+        filled_values[c.cls] = dict()
+
+      if c.value is not None and len(c.value) > 2:
+        if c.value not in filled_values[c.cls]:
+          filled_values[c.cls][c.value] = []
+
+        if c.footprint is not None and len(c.footprint) > 2:
+          # print(c.designator, c.value, c.footprint, c.distributors)
+          tosearch = filled_values[c.cls][c.value]
+          # find element with matching footprint
+          found = None
+          for i, fp in enumerate(tosearch):
+            if fp[0] == c.footprint:
+              found = i
+              break
+
+          if found is None:
+            distributors = []
+            for dist in c.distributors:
+              if len(c.distributors[dist].id) > 2 and len(dist) > 2:
+                distributors.append((dist, c.distributors[dist].id))
+            tosearch.append((c.footprint, [c.designator], distributors))
+          else:
+            tosearch[i][1].append(c.designator)
+            # append any new distributors
+            for dist in c.distributors:
+              if len(c.distributors[dist].id) > 2 and len(dist) > 2:
+                if not any([dist == m[0] and c.distributors[dist].id == m[1] 
+                    for m in tosearch[i][2]]):
+                  tosearch[i][2].append((dist, c.distributors[dist].id))
+  return filled_values
+
+def main():
+  parser = argparse.ArgumentParser(description="Autofills components in an Eeschema schematic")
+  parser.add_argument("-i", "--include", action="append", help="additional files to read in, for component inference")
+  parser.add_argument("input", help="file to autofill")
+  parser.add_argument("output", help="file to write autofilled schematic to")
+
+  args = parser.parse_args()
+
+  lines = None
+  components = None
+
+  with open(args.input, 'r') as f:
+    (lines, components) = parse_lines(f)
+
+  print("{} lines".format(len(lines)))
+  print("found {} components".format(len(components)))
+  without_footprints = len([None for c in components if c.footprint is None])
+  print("found {} components without footprints".format(without_footprints))
+
+  all_components = []
+  all_components.extend(components)
+
+  if args.include is not None:
+    print("searching additional files: {}".format(" ".join(args.include)))
+
+    for filename in args.include:
+      with open(filename, 'r') as f:
+        (more_lines, more_components) = parse_lines(f)
+      print("{} more lines".format(len(more_lines)))
+      print("found {} more components".format(len(more_components)))
+      more_without_footprints = len([None for c in more_components if c.footprint is None])
+      print("found {} more components without footprints".format(more_without_footprints))
+      all_components.extend(more_components)
+
+  filled_values = infer_components(all_components)
+
+  entry_count = 0
+  conflicts = []
+  for cls in filled_values:
+    for val in filled_values[cls]:
+      for _ in filled_values[cls][val]:
+        # TODO: check for conflicts
+        entry_count += 1
+  print("found {} filled unique component classes".format(entry_count))
+
+  # TODO: allow interactive distributor choice to resolve conflicts
+  while len(conflicts) > 0:
+    print("found conflicting information, cannot autofill")
+    for conflict in conflicts:
+      pass
+    return
+
+  # autofill
+  autofill_fp = 0
+  autofill_dist = 0
+  to_append = []
+  for c in components:
+    if c.cls in filled_values:
+      if c.value in filled_values[c.cls]:
+        matches = []
+        if c.footprint is not None:
+          matches = [t for t in filled_values[c.cls][c.value] if t[0] == c.footprint]
+        else:
+          matches = filled_values[c.cls][c.value]
+        if len(matches) == 1:
+          # autofill
+          match = matches[0]
+          if ((c.footprint is None or len(c.footprint) <= 2) and 
+              c.footprint_row is not None):
+            print("matched {} {} with {}".format(c.designator, c.value, match))
+            # rewrite footprint
+            c.footprint = match[0]
+            row = c.footprint_row[1]
+            row[2] = match[0]
+            lines[c.footprint_row[0]] = " ".join(row)
+            autofill_fp += 1
+          # add in distributors 
+          dist_added = 0
+          for dist in match[2]:
+            if dist[0] not in c.distributors:
+              c.distributors[dist[0]] = DistributorData(dist[1])
+              # append to the field list
+              template_row = quotesplit(lines[c.last_value][:-1])
+              row = [
+                  template_row[0],
+                  str(c.num_fields),
+                  dist[1]] + template_row[3:11] + [dist[0] + "\n"]
+              c.num_fields += 1
+              to_append.append((c.last_value, row))
+              dist_added += 1
+              # mark as something to do, because appending now would
+              # cause the row numbers to shift for all the components after this one,
+              # invalidating their indices
+          if dist_added > 0:
+            autofill_dist += 1
+
+    # print(c.designator, c.value, c.footprint, c.distributors)
+  for ta in reversed(to_append):
+    idx = ta[0]
+    row = ta[1]
+    lines = lines[0:idx+1] + [" ".join(row)] + lines[idx+1:]
+
+  print("autofilled {} fp, {} dist".format(autofill_fp, autofill_dist))
+  # dictionary of dictionaries of components without footprints
+  # missing[cls][value] = [designators]
+  missing = dict()
+
+  for c in components:
+    if c.cls is not None:
+      if c.cls not in missing:
+        missing[c.cls] = dict()
+      if c.value is not None and c.footprint is None:
+        if c.value not in missing[c.cls]:
+          missing[c.cls][c.value] = []
+        missing[c.cls][c.value].append(c.designator)
+
+  for cls in missing:
+    for value in missing[cls]:
+      print("NOTE: no unique footprint found for {} {}".format(value, missing[cls][value]))
+
+  # repeat for manufacturer info
+  missing = dict()
+
+  for c in components:
+    if c.cls is not None:
+      if c.cls not in missing:
+        missing[c.cls] = dict()
+      if c.value is not None and c.value != "\"DNP\"" and not bool(c.distributors):
+        if c.value not in missing[c.cls]:
+          missing[c.cls][c.value] = []
+        missing[c.cls][c.value].append(c.designator)
+
+  for cls in missing:
+    for value in missing[cls]:
+      print("NOTE: no distributor data found for {} {}".format(value, missing[cls][value]))
+
+
+  output = args.output or args.input
+  print("outputting to {}...".format(output))
+  with open(output, "w+") as f:
+    for line in lines:
+      f.write(line)
+
+if __name__ == "__main__":
+  main()
diff --git a/pcb/iss-lna/iss-lna.bak b/pcb/iss-lna/iss-lna.bak
index c5e668a..9a59d19 100644
--- a/pcb/iss-lna/iss-lna.bak
+++ b/pcb/iss-lna/iss-lna.bak
@@ -20,7 +20,7 @@ U 1 1 5D244E7D
 P 9050 4050
 F 0 "J4" H 9060 4170 50  0000 C CNN
 F 1 "SMA" V 9165 4050 50  0000 C CNN
-F 2 "" H 9050 4050 50  0001 C CNN
+F 2 "iss_lna:142-0701-801" H 9050 4050 50  0001 C CNN
 F 3 " ~" H 9050 4050 50  0001 C CNN
 	1    9050 4050
 	1    0    0    -1  
@@ -31,7 +31,7 @@ U 1 1 5D24503E
 P 1400 4400
 F 0 "J1" H 1410 4520 50  0000 C CNN
 F 1 "SMA" V 1515 4400 50  0000 C CNN
-F 2 "" H 1400 4400 50  0001 C CNN
+F 2 "iss_lna:142-0701-801" H 1400 4400 50  0001 C CNN
 F 3 " ~" H 1400 4400 50  0001 C CNN
 F 4 " 530-142-0701-801 " H 1400 4400 50  0001 C CNN "Mouser"
 	1    1400 4400
diff --git a/pcb/iss-lna/iss-lna.sch b/pcb/iss-lna/iss-lna.sch
index c2a5c5b..9055e31 100644
--- a/pcb/iss-lna/iss-lna.sch
+++ b/pcb/iss-lna/iss-lna.sch
@@ -20,8 +20,9 @@ U 1 1 5D244E7D
 P 9050 4050
 F 0 "J4" H 9060 4170 50  0000 C CNN
 F 1 "SMA" V 9165 4050 50  0000 C CNN
-F 2 "" H 9050 4050 50  0001 C CNN
+F 2 "iss_lna:142-0701-801" H 9050 4050 50  0001 C CNN
 F 3 " ~" H 9050 4050 50  0001 C CNN
+F 4 " 530-142-0701-801 " H 9050 4050 50  0001 C CNN "Mouser"
 	1    9050 4050
 	1    0    0    -1  
 $EndComp
@@ -361,8 +362,9 @@ U 1 1 5D2A395F
 P 4800 4900
 F 0 "J2" H 4810 5020 50  0000 C CNN
 F 1 "UFL" V 4915 4900 50  0000 C CNN
-F 2 "" H 4800 4900 50  0001 C CNN
+F 2 "iss_lna:2337019-1" H 4800 4900 50  0001 C CNN
 F 3 " ~" H 4800 4900 50  0001 C CNN
+F 4 "571-2337019-1" H 4800 4900 50  0001 C CNN "Mouser"
 	1    4800 4900
 	-1   0    0    -1  
 $EndComp
@@ -400,6 +402,7 @@ F 0 "C10" H 7310 4420 50  0000 L CNN
 F 1 "DNP" H 7310 4270 50  0000 L CNN
 F 2 "Capacitor_SMD:C_0402_1005Metric" H 7300 4350 50  0001 C CNN
 F 3 "~" H 7300 4350 50  0001 C CNN
+F 4 "NoPart" H 7300 4350 50  0001 C CNN "Mouser"
 	1    7300 4350
 	1    0    0    -1  
 $EndComp
@@ -450,6 +453,7 @@ F 0 "C3" H 3110 4720 50  0000 L CNN
 F 1 "DNP" H 3110 4570 50  0000 L CNN
 F 2 "Capacitor_SMD:C_0402_1005Metric" H 3100 4650 50  0001 C CNN
 F 3 "~" H 3100 4650 50  0001 C CNN
+F 4 "NoPart" H 3100 4650 50  0001 C CNN "Mouser"
 	1    3100 4650
 	1    0    0    -1  
 $EndComp
@@ -532,6 +536,7 @@ F 0 "C6" H 4360 4720 50  0000 L CNN
 F 1 "DNP" H 4360 4570 50  0000 L CNN
 F 2 "Capacitor_SMD:C_0402_1005Metric" H 4350 4650 50  0001 C CNN
 F 3 "~" H 4350 4650 50  0001 C CNN
+F 4 "NoPart" H 4350 4650 50  0001 C CNN "Mouser"
 	1    4350 4650
 	1    0    0    -1  
 $EndComp
@@ -554,6 +559,7 @@ F 0 "C5" H 3810 4720 50  0000 L CNN
 F 1 "DNP" H 3810 4570 50  0000 L CNN
 F 2 "Capacitor_SMD:C_0402_1005Metric" H 3800 4650 50  0001 C CNN
 F 3 "~" H 3800 4650 50  0001 C CNN
+F 4 "NoPart" H 3800 4650 50  0001 C CNN "Mouser"
 	1    3800 4650
 	1    0    0    -1  
 $EndComp
@@ -580,6 +586,7 @@ F 0 "C11" H 7960 4370 50  0000 L CNN
 F 1 "DNP" H 7960 4220 50  0000 L CNN
 F 2 "Capacitor_SMD:C_0402_1005Metric" H 7950 4300 50  0001 C CNN
 F 3 "~" H 7950 4300 50  0001 C CNN
+F 4 "NoPart" H 7950 4300 50  0001 C CNN "Mouser"
 	1    7950 4300
 	1    0    0    -1  
 $EndComp
@@ -624,6 +631,7 @@ F 0 "C2" H 2660 4720 50  0000 L CNN
 F 1 "DNP" H 2660 4570 50  0000 L CNN
 F 2 "Capacitor_SMD:C_0402_1005Metric" H 2650 4650 50  0001 C CNN
 F 3 "~" H 2650 4650 50  0001 C CNN
+F 4 "NoPart" H 2650 4650 50  0001 C CNN "Mouser"
 	1    2650 4650
 	1    0    0    -1  
 $EndComp
@@ -772,8 +780,9 @@ U 1 1 5D3AC327
 P 6800 4850
 F 0 "J3" H 6810 4970 50  0000 C CNN
 F 1 "UFL" V 6915 4850 50  0000 C CNN
-F 2 "" H 6800 4850 50  0001 C CNN
+F 2 "iss_lna:2337019-1" H 6800 4850 50  0001 C CNN
 F 3 " ~" H 6800 4850 50  0001 C CNN
+F 4 "571-2337019-1" H 6800 4850 50  0001 C CNN "Mouser"
 	1    6800 4850
 	1    0    0    -1  
 $EndComp
@@ -796,7 +805,7 @@ U 1 1 5D3B3A49
 P 4800 6350
 F 0 "J5" H 4810 6470 50  0000 C CNN
 F 1 "UFL" V 4915 6350 50  0000 C CNN
-F 2 "" H 4800 6350 50  0001 C CNN
+F 2 "iss_lna:2337019-1" H 4800 6350 50  0001 C CNN
 F 3 " ~" H 4800 6350 50  0001 C CNN
 F 4 "571-2337019-1" H 4800 6350 50  0001 C CNN "Mouser"
 	1    4800 6350
@@ -821,8 +830,9 @@ U 1 1 5D3BA203
 P 5950 6350
 F 0 "J6" H 5960 6470 50  0000 C CNN
 F 1 "UFL" V 6065 6350 50  0000 C CNN
-F 2 "" H 5950 6350 50  0001 C CNN
+F 2 "iss_lna:2337019-1" H 5950 6350 50  0001 C CNN
 F 3 " ~" H 5950 6350 50  0001 C CNN
+F 4 "571-2337019-1" H 5950 6350 50  0001 C CNN "Mouser"
 	1    5950 6350
 	1    0    0    -1  
 $EndComp