Browse Source

Fix bug with colons in comments

Ian Adam Naval 1 year ago
parent
commit
d5b968b669
5 changed files with 263 additions and 117 deletions
  1. 1
    0
      Pipfile
  2. 201
    114
      Pipfile.lock
  3. 4
    0
      README.md
  4. 7
    3
      autopilot/scrape.py
  5. 50
    0
      autopilot/test/test_scrape.py

+ 1
- 0
Pipfile View File

@@ -5,6 +5,7 @@ verify_ssl = true
5 5
 
6 6
 [dev-packages]
7 7
 pylint = "*"
8
+pytest = "*"
8 9
 
9 10
 [packages]
10 11
 requests = "*"

+ 201
- 114
Pipfile.lock View File

@@ -1,7 +1,7 @@
1 1
 {
2 2
     "_meta": {
3 3
         "hash": {
4
-            "sha256": "92eba78c05de5de1513dc9d15b0e277d6607a3ca36630adc2f6fdeeec50c98b2"
4
+            "sha256": "202aed259f8823caeba8c11a306429be1098c6bda75ad74676ab0302ebfb46aa"
5 5
         },
6 6
         "pipfile-spec": 6,
7 7
         "requires": {
@@ -18,19 +18,19 @@
18 18
     "default": {
19 19
         "beautifulsoup4": {
20 20
             "hashes": [
21
-                "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858",
22
-                "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348",
23
-                "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718"
21
+                "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
22
+                "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
23
+                "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
24 24
             ],
25 25
             "index": "pypi",
26
-            "version": "==4.7.1"
26
+            "version": "==4.8.0"
27 27
         },
28 28
         "certifi": {
29 29
             "hashes": [
30
-                "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
31
-                "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
30
+                "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
31
+                "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
32 32
             ],
33
-            "version": "==2018.11.29"
33
+            "version": "==2019.9.11"
34 34
         },
35 35
         "chardet": {
36 36
             "hashes": [
@@ -48,11 +48,11 @@
48 48
         },
49 49
         "flask": {
50 50
             "hashes": [
51
-                "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48",
52
-                "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05"
51
+                "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52",
52
+                "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6"
53 53
             ],
54 54
             "index": "pypi",
55
-            "version": "==1.0.2"
55
+            "version": "==1.1.1"
56 56
         },
57 57
         "idna": {
58 58
             "hashes": [
@@ -70,78 +70,77 @@
70 70
         },
71 71
         "jinja2": {
72 72
             "hashes": [
73
-                "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd",
74
-                "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"
73
+                "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013",
74
+                "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b"
75 75
             ],
76
-            "version": "==2.10"
76
+            "version": "==2.10.1"
77 77
         },
78 78
         "markupsafe": {
79 79
             "hashes": [
80
-                "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432",
81
-                "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b",
82
-                "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9",
83
-                "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af",
84
-                "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834",
85
-                "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd",
86
-                "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d",
87
-                "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7",
88
-                "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b",
89
-                "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3",
90
-                "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c",
91
-                "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2",
92
-                "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7",
93
-                "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36",
94
-                "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1",
95
-                "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e",
96
-                "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1",
97
-                "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c",
98
-                "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856",
99
-                "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550",
100
-                "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492",
101
-                "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672",
102
-                "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401",
103
-                "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6",
104
-                "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6",
105
-                "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c",
106
-                "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd",
107
-                "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1"
80
+                "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
81
+                "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
82
+                "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
83
+                "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
84
+                "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
85
+                "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
86
+                "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
87
+                "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
88
+                "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
89
+                "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
90
+                "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
91
+                "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
92
+                "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
93
+                "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
94
+                "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
95
+                "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
96
+                "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",
97
+                "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",
98
+                "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",
99
+                "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",
100
+                "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",
101
+                "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",
102
+                "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",
103
+                "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",
104
+                "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
105
+                "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
106
+                "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
107
+                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"
108 108
             ],
109
-            "version": "==1.1.0"
109
+            "version": "==1.1.1"
110 110
         },
111 111
         "pendulum": {
112 112
             "hashes": [
113
-                "sha256:0f43d963b27e92b04047ce8352e4c277db99f20d0b513df7d0ceafe674a2f727",
114
-                "sha256:14e60d26d7400980123dbb6e3f2a90b70d7c18c63742ffe5bd6d6a643f8c6ef1",
115
-                "sha256:5035a4e17504814a679f138374269cc7cc514aeac7ba6d9dc020abc224f25dbc",
116
-                "sha256:8c0b3d655c1e9205d4dacf42fffc929cde3b19b5fb544a7f7561e6896eb8a000",
117
-                "sha256:bfc7b33ae193a204ec0bec12ad0d2d3300cd7e51d91d992da525ba3b28f0d265",
118
-                "sha256:cd70b75800439794e1ad8dbfa24838845e171918df81fa98b68d0d5a6f9b8bf2",
119
-                "sha256:cf535d36c063575d4752af36df928882b2e0e31541b4482c97d63752785f9fcb"
113
+                "sha256:1cde6e3c6310fb882c98f373795f807cb2bd6af01f34d2857e6e283b5ee91e09",
114
+                "sha256:485aef2089defee88607d37d5bc238934d0b90993d7bf9ceb36e481af41e9c66",
115
+                "sha256:57801754e05f30e8a7e4d24734c9fad82c6c3ec489151555f0fc66bb32ba6d6d",
116
+                "sha256:7ee344bc87cb425b04717b90d14ffde14c1dd64eaa73060b3772edcf57f3e866",
117
+                "sha256:c460f4d8dc41ec3c4377ac1807678cd72fe5e973cc2943c104ffdeaac32dacb7",
118
+                "sha256:d3078e007315a959989c41cee5cfd63cfeeca21dd3d8295f4bc24199489e9b6c"
120 119
             ],
121 120
             "index": "pypi",
122
-            "version": "==2.0.4"
121
+            "version": "==2.0.5"
123 122
         },
124 123
         "python-dateutil": {
125 124
             "hashes": [
126
-                "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
127
-                "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
125
+                "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
126
+                "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
128 127
             ],
129
-            "version": "==2.7.5"
128
+            "version": "==2.8.0"
130 129
         },
131 130
         "pytzdata": {
132 131
             "hashes": [
133
-                "sha256:10c74b0cfc51a9269031f86ecd11096c9c6a141f5bb15a3b8a88f9979f6361e2",
134
-                "sha256:279cbd9900d5da9a8f9053e60db0db7f42d9a799673744b76aaeb6b4f14abe77"
132
+                "sha256:84c52b9a47d097fcd483f047a544979de6c3a86e94c845e3569e9f8acd0fa071",
133
+                "sha256:fac06f7cdfa903188dc4848c655e4adaee67ee0f2fe08e7daf815cf2a761ee5e"
135 134
             ],
136
-            "version": "==2018.7"
135
+            "version": "==2019.3"
137 136
         },
138 137
         "requests": {
139 138
             "hashes": [
140
-                "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
141
-                "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
139
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
140
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
142 141
             ],
143 142
             "index": "pypi",
144
-            "version": "==2.21.0"
143
+            "version": "==2.22.0"
145 144
         },
146 145
         "six": {
147 146
             "hashes": [
@@ -152,24 +151,24 @@
152 151
         },
153 152
         "soupsieve": {
154 153
             "hashes": [
155
-                "sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270",
156
-                "sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b"
154
+                "sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3",
155
+                "sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6"
157 156
             ],
158
-            "version": "==1.7.2"
157
+            "version": "==1.9.4"
159 158
         },
160 159
         "urllib3": {
161 160
             "hashes": [
162
-                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
163
-                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
161
+                "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
162
+                "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
164 163
             ],
165
-            "version": "==1.24.1"
164
+            "version": "==1.25.6"
166 165
         },
167 166
         "uwsgi": {
168 167
             "hashes": [
169
-                "sha256:d2318235c74665a60021a4fc7770e9c2756f9fc07de7b8c22805efe85b5ab277"
168
+                "sha256:4972ac538800fb2d421027f49b4a1869b66048839507ccf0aa2fda792d99f583"
170 169
             ],
171 170
             "index": "pypi",
172
-            "version": "==2.0.17.1"
171
+            "version": "==2.0.18"
173 172
         },
174 173
         "vobject": {
175 174
             "hashes": [
@@ -180,61 +179,71 @@
180 179
         },
181 180
         "werkzeug": {
182 181
             "hashes": [
183
-                "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c",
184
-                "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
182
+                "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7",
183
+                "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"
185 184
             ],
186
-            "version": "==0.14.1"
185
+            "version": "==0.16.0"
187 186
         }
188 187
     },
189 188
     "develop": {
190 189
         "astroid": {
191 190
             "hashes": [
192
-                "sha256:35b032003d6a863f5dcd7ec11abd5cd5893428beaa31ab164982403bcb311f22",
193
-                "sha256:6a5d668d7dc69110de01cdf7aeec69a679ef486862a0850cc0fd5571505b6b7e"
191
+                "sha256:9b3f17b0550f82e28a6776a4e5222441f48e523b0773df4bc505bb6b7c2093b7",
192
+                "sha256:c7e2e5773d87ccc00d01c273e439386f4d6d63cce61317a79ccce5880162f9fb"
193
+            ],
194
+            "version": "==2.3.0"
195
+        },
196
+        "atomicwrites": {
197
+            "hashes": [
198
+                "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
199
+                "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
200
+            ],
201
+            "version": "==1.3.0"
202
+        },
203
+        "attrs": {
204
+            "hashes": [
205
+                "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
206
+                "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
207
+            ],
208
+            "version": "==19.1.0"
209
+        },
210
+        "importlib-metadata": {
211
+            "hashes": [
212
+                "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
213
+                "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
194 214
             ],
195
-            "version": "==2.1.0"
215
+            "markers": "python_version < '3.8'",
216
+            "version": "==0.23"
196 217
         },
197 218
         "isort": {
198 219
             "hashes": [
199
-                "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af",
200
-                "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8",
201
-                "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497"
220
+                "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
221
+                "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
202 222
             ],
203
-            "version": "==4.3.4"
223
+            "version": "==4.3.21"
204 224
         },
205 225
         "lazy-object-proxy": {
206 226
             "hashes": [
207
-                "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33",
208
-                "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39",
209
-                "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019",
210
-                "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088",
211
-                "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b",
212
-                "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e",
213
-                "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6",
214
-                "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b",
215
-                "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5",
216
-                "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff",
217
-                "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd",
218
-                "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7",
219
-                "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff",
220
-                "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d",
221
-                "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2",
222
-                "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35",
223
-                "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4",
224
-                "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514",
225
-                "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252",
226
-                "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109",
227
-                "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f",
228
-                "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c",
229
-                "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92",
230
-                "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577",
231
-                "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d",
232
-                "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d",
233
-                "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f",
234
-                "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a",
235
-                "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b"
236
-            ],
237
-            "version": "==1.3.1"
227
+                "sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf",
228
+                "sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3",
229
+                "sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce",
230
+                "sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f",
231
+                "sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f",
232
+                "sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0",
233
+                "sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e",
234
+                "sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905",
235
+                "sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8",
236
+                "sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2",
237
+                "sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009",
238
+                "sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a",
239
+                "sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512",
240
+                "sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5",
241
+                "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e",
242
+                "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4",
243
+                "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f",
244
+                "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1"
245
+            ],
246
+            "version": "==1.4.2"
238 247
         },
239 248
         "mccabe": {
240 249
             "hashes": [
@@ -243,13 +252,56 @@
243 252
             ],
244 253
             "version": "==0.6.1"
245 254
         },
255
+        "more-itertools": {
256
+            "hashes": [
257
+                "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
258
+                "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
259
+            ],
260
+            "version": "==7.2.0"
261
+        },
262
+        "packaging": {
263
+            "hashes": [
264
+                "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
265
+                "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
266
+            ],
267
+            "version": "==19.2"
268
+        },
269
+        "pluggy": {
270
+            "hashes": [
271
+                "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
272
+                "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
273
+            ],
274
+            "version": "==0.13.0"
275
+        },
276
+        "py": {
277
+            "hashes": [
278
+                "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
279
+                "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
280
+            ],
281
+            "version": "==1.8.0"
282
+        },
246 283
         "pylint": {
247 284
             "hashes": [
248
-                "sha256:689de29ae747642ab230c6d37be2b969bf75663176658851f456619aacf27492",
249
-                "sha256:771467c434d0d9f081741fec1d64dfb011ed26e65e12a28fe06ca2f61c4d556c"
285
+                "sha256:2d64b4b8fa044480b1a49d47535da53557f8f426b8c5bd6a23beb65e905101a1",
286
+                "sha256:6cbd124a1a5ed1fd3f3fed4178a6c2ba166862ea0dac6ab2ff8d9f0998b13e5c"
287
+            ],
288
+            "index": "pypi",
289
+            "version": "==2.4.1"
290
+        },
291
+        "pyparsing": {
292
+            "hashes": [
293
+                "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
294
+                "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
295
+            ],
296
+            "version": "==2.4.2"
297
+        },
298
+        "pytest": {
299
+            "hashes": [
300
+                "sha256:13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5",
301
+                "sha256:d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5"
250 302
             ],
251 303
             "index": "pypi",
252
-            "version": "==2.2.2"
304
+            "version": "==5.2.0"
253 305
         },
254 306
         "six": {
255 307
             "hashes": [
@@ -258,11 +310,46 @@
258 310
             ],
259 311
             "version": "==1.12.0"
260 312
         },
313
+        "typed-ast": {
314
+            "hashes": [
315
+                "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
316
+                "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e",
317
+                "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0",
318
+                "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c",
319
+                "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631",
320
+                "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4",
321
+                "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34",
322
+                "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b",
323
+                "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a",
324
+                "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233",
325
+                "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1",
326
+                "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36",
327
+                "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d",
328
+                "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
329
+                "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
330
+            ],
331
+            "markers": "implementation_name == 'cpython' and python_version >= '3.7' and python_version < '3.8'",
332
+            "version": "==1.4.0"
333
+        },
334
+        "wcwidth": {
335
+            "hashes": [
336
+                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
337
+                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
338
+            ],
339
+            "version": "==0.1.7"
340
+        },
261 341
         "wrapt": {
262 342
             "hashes": [
263
-                "sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533"
343
+                "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
344
+            ],
345
+            "version": "==1.11.2"
346
+        },
347
+        "zipp": {
348
+            "hashes": [
349
+                "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
350
+                "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
264 351
             ],
265
-            "version": "==1.11.1"
352
+            "version": "==0.6.0"
266 353
         }
267 354
     }
268 355
 }

+ 4
- 0
README.md View File

@@ -14,6 +14,10 @@ $EDITOR .env  # add your config
14 14
 make build
15 15
 ```
16 16
 
17
+## Tests
18
+
19
+Running tests is currently done outside of Docker at this time.
20
+
17 21
 ## Usage
18 22
 
19 23
 Run locally with Docker:

+ 7
- 3
autopilot/scrape.py View File

@@ -63,6 +63,10 @@ def raw_schedule_html(session: requests.Session) -> BeautifulSoup:
63 63
     return response.text
64 64
 
65 65
 
66
+def _parse_comments(lines: List[str]) -> str:
67
+    return "\n".join(line for line in lines if line) + "\n"
68
+
69
+
66 70
 def make_reservation_from_tag(tag: Tag) -> Reservation:
67 71
     """Parse attributes from the title attribute of the reservation
68 72
 
@@ -75,16 +79,16 @@ def make_reservation_from_tag(tag: Tag) -> Reservation:
75 79
     Start		: 1/26/2019 12:00 PM
76 80
     End		: 1/26/2019 3:00 PM
77 81
     Comments -----------------------------
78
-    "
82
+
79 83
     """
80 84
     raw = tag.get('title')
81 85
     lines = raw.replace('\t', '').split('\r\n')
82 86
     attributes = {}
83
-    for line in lines:
87
+    for line in lines[:6]:
84 88
         if ':' in line:
85 89
             key, value = line.split(':', maxsplit=1)
86 90
             attributes[key.lower().strip()] = value.strip()
87
-    attributes['comments'] = lines[-2]
91
+    attributes['comments'] = _parse_comments(lines[6:])
88 92
     for datetime_attr in {'start', 'end'}:
89 93
         parsed_datetime = pendulum.parse(attributes[datetime_attr], 
90 94
                                             strict=False,

+ 50
- 0
autopilot/test/test_scrape.py View File

@@ -0,0 +1,50 @@
1
+import pendulum
2
+from bs4 import Tag
3
+
4
+from autopilot import config
5
+from autopilot.scrape import make_reservation_from_tag
6
+
7
+
8
+class TestMakeReservationFromTag(object):
9
+
10
+    def test_blank_comment(self):
11
+        title = (
12
+            "Pilot\t: Ian Naval\r\n"
13
+            "Instructor\t: Agnelo Lopes CFI, CFII\r\n"
14
+            "Aircraft\t: N222ND\r\n"
15
+            "Location\t: Bedford\r\n"
16
+            "Start\t: 1/26/2019 12:00 PM\r\n"
17
+            "End\t: 1/26/2019 3:00 PM\r\n"
18
+            "Comments -----------------------------\r\n"
19
+        )
20
+        tag = Tag(name='div', attrs={'title': title})
21
+        reservation = make_reservation_from_tag(tag)
22
+        assert reservation.pilot == 'Ian Naval'
23
+        assert reservation.instructor == 'Agnelo Lopes CFI, CFII'
24
+        assert reservation.aircraft == 'N222ND'
25
+        assert reservation.location == 'Bedford'
26
+        assert reservation.start == pendulum.parse(
27
+            '1/26/2019 12:00 PM', strict=False, tz=config.TIME_ZONE)
28
+        assert reservation.end == pendulum.parse(
29
+            '1/26/2019 3:00 PM', strict=False, tz=config.TIME_ZONE)
30
+        assert reservation.comments == (
31
+            "Comments -----------------------------\n"
32
+        )
33
+
34
+    def test_multiline_comment(self):
35
+        title = (
36
+            "Pilot\t: Ian Naval\r\n"
37
+            "Instructor\t: Agnelo Lopes CFI, CFII\r\n"
38
+            "Aircraft\t: N222ND\r\n"
39
+            "Location\t: Bedford\r\n"
40
+            "Start\t: 1/26/2019 12:00 PM\r\n"
41
+            "End\t: 1/26/2019 3:00 PM\r\n"
42
+            "Comments -----------------------------\r\n"
43
+            "11:00 - Depart Hanscom\r\n"
44
+        )
45
+        tag = Tag(name='div', attrs={'title': title})
46
+        reservation = make_reservation_from_tag(tag)
47
+        assert reservation.comments == (
48
+            "Comments -----------------------------\n"
49
+            "11:00 - Depart Hanscom\n"
50
+        )

Loading…
Cancel
Save