common: Drop linux/delay.h from common header
[oweals/u-boot.git] / drivers / net / ldpaa_eth / ldpaa_eth.c
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Copyright 2014-2016 Freescale Semiconductor, Inc.
4  * Copyright 2017 NXP
5  */
6
7 #include <common.h>
8 #include <cpu_func.h>
9 #include <log.h>
10 #include <asm/io.h>
11 #include <asm/types.h>
12 #include <malloc.h>
13 #include <net.h>
14 #include <hwconfig.h>
15 #include <phy.h>
16 #include <miiphy.h>
17 #include <linux/bug.h>
18 #include <linux/compat.h>
19 #include <fsl-mc/fsl_dpmac.h>
20 #include <linux/delay.h>
21
22 #include <fsl-mc/ldpaa_wriop.h>
23 #include "ldpaa_eth.h"
24
25 #ifdef CONFIG_PHYLIB
26 #ifdef CONFIG_DM_ETH
27 static void init_phy(struct udevice *dev)
28 {
29         struct ldpaa_eth_priv *priv = dev_get_priv(dev);
30
31         priv->phy = dm_eth_phy_connect(dev);
32
33         if (!priv->phy)
34                 return;
35
36         phy_config(priv->phy);
37 }
38 #else
39 static int init_phy(struct eth_device *dev)
40 {
41         struct ldpaa_eth_priv *priv = (struct ldpaa_eth_priv *)dev->priv;
42         struct phy_device *phydev = NULL;
43         struct mii_dev *bus;
44         int phy_addr, phy_num;
45         int ret = 0;
46
47         bus = wriop_get_mdio(priv->dpmac_id);
48         if (bus == NULL)
49                 return 0;
50
51         for (phy_num = 0; phy_num < WRIOP_MAX_PHY_NUM; phy_num++) {
52                 phy_addr = wriop_get_phy_address(priv->dpmac_id, phy_num);
53                 if (phy_addr < 0)
54                         continue;
55
56                 phydev = phy_connect(bus, phy_addr, dev,
57                                      wriop_get_enet_if(priv->dpmac_id));
58                 if (!phydev) {
59                         printf("Failed to connect\n");
60                         ret = -ENODEV;
61                         break;
62                 }
63                 wriop_set_phy_dev(priv->dpmac_id, phy_num, phydev);
64                 ret = phy_config(phydev);
65                 if (ret)
66                         break;
67         }
68
69         if (ret) {
70                 for (phy_num = 0; phy_num < WRIOP_MAX_PHY_NUM; phy_num++) {
71                         phydev = wriop_get_phy_dev(priv->dpmac_id, phy_num);
72                         if (!phydev)
73                                 continue;
74
75                         free(phydev);
76                         wriop_set_phy_dev(priv->dpmac_id, phy_num, NULL);
77                 }
78         }
79
80         return ret;
81 }
82 #endif
83 #endif
84
85 #ifdef DEBUG
86
87 #define DPNI_STATS_PER_PAGE 6
88
89 static const char *dpni_statistics[][DPNI_STATS_PER_PAGE] = {
90         {
91         "DPNI_CNT_ING_ALL_FRAMES",
92         "DPNI_CNT_ING_ALL_BYTES",
93         "DPNI_CNT_ING_MCAST_FRAMES",
94         "DPNI_CNT_ING_MCAST_BYTES",
95         "DPNI_CNT_ING_BCAST_FRAMES",
96         "DPNI_CNT_ING_BCAST_BYTES",
97         }, {
98         "DPNI_CNT_EGR_ALL_FRAMES",
99         "DPNI_CNT_EGR_ALL_BYTES",
100         "DPNI_CNT_EGR_MCAST_FRAMES",
101         "DPNI_CNT_EGR_MCAST_BYTES",
102         "DPNI_CNT_EGR_BCAST_FRAMES",
103         "DPNI_CNT_EGR_BCAST_BYTES",
104         }, {
105         "DPNI_CNT_ING_FILTERED_FRAMES",
106         "DPNI_CNT_ING_DISCARDED_FRAMES",
107         "DPNI_CNT_ING_NOBUFFER_DISCARDS",
108         "DPNI_CNT_EGR_DISCARDED_FRAMES",
109         "DPNI_CNT_EGR_CNF_FRAMES",
110         ""
111         },
112 };
113
114 static void print_dpni_stats(const char *strings[],
115                              struct dpni_statistics dpni_stats)
116 {
117         uint64_t *stat;
118         int i;
119
120         stat = (uint64_t *)&dpni_stats;
121         for (i = 0; i < DPNI_STATS_PER_PAGE; i++) {
122                 if (strcmp(strings[i], "\0") == 0)
123                         break;
124                 printf("%s= %llu\n", strings[i], *stat);
125                 stat++;
126         }
127 }
128
129 static void ldpaa_eth_get_dpni_counter(void)
130 {
131         int err = 0;
132         unsigned int page = 0;
133         struct dpni_statistics dpni_stats;
134
135         printf("DPNI counters ..\n");
136         for (page = 0; page < 3; page++) {
137                 err = dpni_get_statistics(dflt_mc_io, MC_CMD_NO_FLAGS,
138                                           dflt_dpni->dpni_handle, page,
139                                           &dpni_stats);
140                 if (err < 0) {
141                         printf("dpni_get_statistics: failed:");
142                         printf("%d for page[%d]\n", err, page);
143                         return;
144                 }
145                 print_dpni_stats(dpni_statistics[page], dpni_stats);
146         }
147 }
148
149 #ifdef CONFIG_DM_ETH
150 static void ldpaa_eth_get_dpmac_counter(struct udevice *dev)
151 {
152         struct ldpaa_eth_priv *priv = dev_get_priv(dev);
153 #else
154 static void ldpaa_eth_get_dpmac_counter(struct eth_device *net_dev)
155 {
156         struct ldpaa_eth_priv *priv = (struct ldpaa_eth_priv *)net_dev->priv;
157 #endif
158         int err = 0;
159         u64 value;
160
161         err = dpmac_get_counter(dflt_mc_io, MC_CMD_NO_FLAGS,
162                      priv->dpmac_handle,
163                      DPMAC_CNT_ING_BYTE,
164                      &value);
165         if (err < 0) {
166                 printf("dpmac_get_counter: DPMAC_CNT_ING_BYTE failed\n");
167                 return;
168         }
169         printf("\nDPMAC counters ..\n");
170         printf("DPMAC_CNT_ING_BYTE=%lld\n", value);
171
172         err = dpmac_get_counter(dflt_mc_io, MC_CMD_NO_FLAGS,
173                      priv->dpmac_handle,
174                      DPMAC_CNT_ING_FRAME_DISCARD,
175                      &value);
176         if (err < 0) {
177                 printf("dpmac_get_counter: DPMAC_CNT_ING_FRAME_DISCARD failed\n");
178                 return;
179         }
180         printf("DPMAC_CNT_ING_FRAME_DISCARD=%lld\n", value);
181
182         err = dpmac_get_counter(dflt_mc_io, MC_CMD_NO_FLAGS,
183                      priv->dpmac_handle,
184                      DPMAC_CNT_ING_ALIGN_ERR,
185                      &value);
186         if (err < 0) {
187                 printf("dpmac_get_counter: DPMAC_CNT_ING_ALIGN_ERR failed\n");
188                 return;
189         }
190         printf("DPMAC_CNT_ING_ALIGN_ERR =%lld\n", value);
191
192         err = dpmac_get_counter(dflt_mc_io, MC_CMD_NO_FLAGS,
193                      priv->dpmac_handle,
194                      DPMAC_CNT_ING_BYTE,
195                      &value);
196         if (err < 0) {
197                 printf("dpmac_get_counter: DPMAC_CNT_ING_BYTE failed\n");
198                 return;
199         }
200         printf("DPMAC_CNT_ING_BYTE=%lld\n", value);
201
202         err = dpmac_get_counter(dflt_mc_io, MC_CMD_NO_FLAGS,
203                      priv->dpmac_handle,
204                      DPMAC_CNT_ING_ERR_FRAME,
205                      &value);
206         if (err < 0) {
207                 printf("dpmac_get_counter: DPMAC_CNT_ING_ERR_FRAME failed\n");
208                 return;
209         }
210         printf("DPMAC_CNT_ING_ERR_FRAME=%lld\n", value);
211
212         err = dpmac_get_counter(dflt_mc_io, MC_CMD_NO_FLAGS,
213                      priv->dpmac_handle,
214                      DPMAC_CNT_EGR_BYTE ,
215                      &value);
216         if (err < 0) {
217                 printf("dpmac_get_counter: DPMAC_CNT_EGR_BYTE failed\n");
218                 return;
219         }
220         printf("DPMAC_CNT_EGR_BYTE =%lld\n", value);
221
222         err = dpmac_get_counter(dflt_mc_io, MC_CMD_NO_FLAGS,
223                      priv->dpmac_handle,
224                      DPMAC_CNT_EGR_ERR_FRAME ,
225                      &value);
226         if (err < 0) {
227                 printf("dpmac_get_counter: DPMAC_CNT_EGR_ERR_FRAME failed\n");
228                 return;
229         }
230         printf("DPMAC_CNT_EGR_ERR_FRAME =%lld\n", value);
231 }
232 #endif
233
234 static void ldpaa_eth_rx(struct ldpaa_eth_priv *priv,
235                          const struct dpaa_fd *fd)
236 {
237         u64 fd_addr;
238         uint16_t fd_offset;
239         uint32_t fd_length;
240         struct ldpaa_fas *fas;
241         uint32_t status, err;
242         u32 timeo = (CONFIG_SYS_HZ * 2) / 1000;
243         u32 time_start;
244         struct qbman_release_desc releasedesc;
245         struct qbman_swp *swp = dflt_dpio->sw_portal;
246
247         fd_addr = ldpaa_fd_get_addr(fd);
248         fd_offset = ldpaa_fd_get_offset(fd);
249         fd_length = ldpaa_fd_get_len(fd);
250
251         debug("Rx frame:data addr=0x%p size=0x%x\n", (u64 *)fd_addr, fd_length);
252
253         if (fd->simple.frc & LDPAA_FD_FRC_FASV) {
254                 /* Read the frame annotation status word and check for errors */
255                 fas = (struct ldpaa_fas *)
256                                 ((uint8_t *)(fd_addr) +
257                                 dflt_dpni->buf_layout.private_data_size);
258                 status = le32_to_cpu(fas->status);
259                 if (status & LDPAA_ETH_RX_ERR_MASK) {
260                         printf("Rx frame error(s): 0x%08x\n",
261                                status & LDPAA_ETH_RX_ERR_MASK);
262                         goto error;
263                 } else if (status & LDPAA_ETH_RX_UNSUPP_MASK) {
264                         printf("Unsupported feature in bitmask: 0x%08x\n",
265                                status & LDPAA_ETH_RX_UNSUPP_MASK);
266                         goto error;
267                 }
268         }
269
270         debug("Rx frame: To Upper layer\n");
271         net_process_received_packet((uint8_t *)(fd_addr) + fd_offset,
272                                     fd_length);
273
274 error:
275         flush_dcache_range(fd_addr, fd_addr + LDPAA_ETH_RX_BUFFER_SIZE);
276         qbman_release_desc_clear(&releasedesc);
277         qbman_release_desc_set_bpid(&releasedesc, dflt_dpbp->dpbp_attr.bpid);
278         time_start = get_timer(0);
279         do {
280                 /* Release buffer into the QBMAN */
281                 err = qbman_swp_release(swp, &releasedesc, &fd_addr, 1);
282         } while (get_timer(time_start) < timeo && err == -EBUSY);
283
284         if (err == -EBUSY)
285                 printf("Rx frame: QBMAN buffer release fails\n");
286
287         return;
288 }
289
290 #ifdef CONFIG_DM_ETH
291 static int ldpaa_eth_pull_dequeue_rx(struct udevice *dev,
292                                      int flags, uchar **packetp)
293 {
294         struct ldpaa_eth_priv *priv = dev_get_priv(dev);
295 #else
296 static int ldpaa_eth_pull_dequeue_rx(struct eth_device *dev)
297 {
298         struct ldpaa_eth_priv *priv = (struct ldpaa_eth_priv *)dev->priv;
299 #endif
300         const struct ldpaa_dq *dq;
301         const struct dpaa_fd *fd;
302         int i = 5, err = 0, status;
303         u32 timeo = (CONFIG_SYS_HZ * 2) / 1000;
304         u32 time_start;
305         static struct qbman_pull_desc pulldesc;
306         struct qbman_swp *swp = dflt_dpio->sw_portal;
307
308         while (--i) {
309                 qbman_pull_desc_clear(&pulldesc);
310                 qbman_pull_desc_set_numframes(&pulldesc, 1);
311                 qbman_pull_desc_set_fq(&pulldesc, priv->rx_dflt_fqid);
312
313                 err = qbman_swp_pull(swp, &pulldesc);
314                 if (err < 0) {
315                         printf("Dequeue frames error:0x%08x\n", err);
316                         continue;
317                 }
318
319                 time_start = get_timer(0);
320
321                  do {
322                         dq = qbman_swp_dqrr_next(swp);
323                 } while (get_timer(time_start) < timeo && !dq);
324
325                 if (dq) {
326                         /* Check for valid frame. If not sent a consume
327                          * confirmation to QBMAN otherwise give it to NADK
328                          * application and then send consume confirmation to
329                          * QBMAN.
330                          */
331                         status = (uint8_t)ldpaa_dq_flags(dq);
332                         if ((status & LDPAA_DQ_STAT_VALIDFRAME) == 0) {
333                                 debug("Dequeue RX frames:");
334                                 debug("No frame delivered\n");
335
336                                 qbman_swp_dqrr_consume(swp, dq);
337                                 continue;
338                         }
339
340                         fd = ldpaa_dq_fd(dq);
341
342                         /* Obtain FD and process it */
343                         ldpaa_eth_rx(priv, fd);
344                         qbman_swp_dqrr_consume(swp, dq);
345                         break;
346                 } else {
347                         err = -ENODATA;
348                         debug("No DQRR entries\n");
349                         break;
350                 }
351         }
352
353         return err;
354 }
355
356 #ifdef CONFIG_DM_ETH
357 static int ldpaa_eth_tx(struct udevice *dev, void *buf, int len)
358 {
359         struct ldpaa_eth_priv *priv = dev_get_priv(dev);
360 #else
361 static int ldpaa_eth_tx(struct eth_device *net_dev, void *buf, int len)
362 {
363         struct ldpaa_eth_priv *priv = (struct ldpaa_eth_priv *)net_dev->priv;
364 #endif
365         struct dpaa_fd fd;
366         u64 buffer_start;
367         int data_offset, err;
368         u32 timeo = (CONFIG_SYS_HZ * 10) / 1000;
369         u32 time_start;
370         struct qbman_swp *swp = dflt_dpio->sw_portal;
371         struct qbman_eq_desc ed;
372         struct qbman_release_desc releasedesc;
373
374         /* Setup the FD fields */
375         memset(&fd, 0, sizeof(fd));
376
377         data_offset = priv->tx_data_offset;
378
379         do {
380                 err = qbman_swp_acquire(dflt_dpio->sw_portal,
381                                         dflt_dpbp->dpbp_attr.bpid,
382                                         &buffer_start, 1);
383         } while (err == -EBUSY);
384
385         if (err <= 0) {
386                 printf("qbman_swp_acquire() failed\n");
387                 return -ENOMEM;
388         }
389
390         debug("TX data: malloc buffer start=0x%p\n", (u64 *)buffer_start);
391
392         memcpy(((uint8_t *)(buffer_start) + data_offset), buf, len);
393
394         flush_dcache_range(buffer_start, buffer_start +
395                                         LDPAA_ETH_RX_BUFFER_SIZE);
396
397         ldpaa_fd_set_addr(&fd, (u64)buffer_start);
398         ldpaa_fd_set_offset(&fd, (uint16_t)(data_offset));
399         ldpaa_fd_set_bpid(&fd, dflt_dpbp->dpbp_attr.bpid);
400         ldpaa_fd_set_len(&fd, len);
401
402         fd.simple.ctrl = LDPAA_FD_CTRL_ASAL | LDPAA_FD_CTRL_PTA |
403                                 LDPAA_FD_CTRL_PTV1;
404
405         qbman_eq_desc_clear(&ed);
406         qbman_eq_desc_set_no_orp(&ed, 0);
407         qbman_eq_desc_set_qd(&ed, priv->tx_qdid, priv->tx_flow_id, 0);
408
409         time_start = get_timer(0);
410
411         while (get_timer(time_start) < timeo) {
412                 err = qbman_swp_enqueue(swp, &ed,
413                                 (const struct qbman_fd *)(&fd));
414                 if (err != -EBUSY)
415                         break;
416         }
417
418         if (err < 0) {
419                 printf("error enqueueing Tx frame\n");
420                 goto error;
421         }
422
423         return err;
424
425 error:
426         qbman_release_desc_clear(&releasedesc);
427         qbman_release_desc_set_bpid(&releasedesc, dflt_dpbp->dpbp_attr.bpid);
428         time_start = get_timer(0);
429         do {
430                 /* Release buffer into the QBMAN */
431                 err = qbman_swp_release(swp, &releasedesc, &buffer_start, 1);
432         } while (get_timer(time_start) < timeo && err == -EBUSY);
433
434         if (err == -EBUSY)
435                 printf("TX data: QBMAN buffer release fails\n");
436
437         return err;
438 }
439
440 static struct phy_device *ldpaa_get_phydev(struct ldpaa_eth_priv *priv)
441 {
442 #ifdef CONFIG_DM_ETH
443         return priv->phy;
444 #else
445 #ifdef CONFIG_PHYLIB
446         struct phy_device *phydev = NULL;
447         int phy_num;
448
449         /* start the phy devices one by one and update the dpmac state */
450         for (phy_num = 0; phy_num < WRIOP_MAX_PHY_NUM; phy_num++) {
451                 phydev = wriop_get_phy_dev(priv->dpmac_id, phy_num);
452                 if (phydev)
453                         return phydev;
454         }
455         return NULL;
456 #endif
457         return NULL;
458 #endif
459 }
460
461 static int ldpaa_get_dpmac_state(struct ldpaa_eth_priv *priv,
462                                  struct dpmac_link_state *state)
463 {
464         phy_interface_t enet_if;
465         struct phy_device *phydev = NULL;
466         int err;
467
468         /* let's start off with maximum capabilities */
469         enet_if = wriop_get_enet_if(priv->dpmac_id);
470         switch (enet_if) {
471         case PHY_INTERFACE_MODE_XGMII:
472                 state->rate = SPEED_10000;
473                 break;
474         default:
475                 state->rate = SPEED_1000;
476                 break;
477         }
478
479         state->up = 1;
480         state->options |= DPMAC_LINK_OPT_AUTONEG;
481         phydev = ldpaa_get_phydev(priv);
482
483         if (phydev) {
484                 err = phy_startup(phydev);
485                 if (err) {
486                         printf("%s: Could not initialize\n", phydev->dev->name);
487                         state->up = 0;
488                 } else if (phydev->link) {
489                         state->rate = min(state->rate, (uint32_t)phydev->speed);
490                         if (!phydev->duplex)
491                                 state->options |= DPMAC_LINK_OPT_HALF_DUPLEX;
492                         if (!phydev->autoneg)
493                                 state->options &= ~DPMAC_LINK_OPT_AUTONEG;
494                 } else {
495                         state->up = 0;
496                 }
497         }
498
499         if (!phydev)
500                 state->options &= ~DPMAC_LINK_OPT_AUTONEG;
501
502         if (!state->up) {
503                 state->rate = 0;
504                 state->options = 0;
505                 return -ENOLINK;
506         }
507
508         return 0;
509 }
510
511 #ifdef CONFIG_DM_ETH
512 static int ldpaa_eth_open(struct udevice *dev)
513 {
514         struct eth_pdata *plat = dev_get_platdata(dev);
515         struct ldpaa_eth_priv *priv = dev_get_priv(dev);
516 #else
517 static int ldpaa_eth_open(struct eth_device *net_dev, bd_t *bd)
518 {
519         struct ldpaa_eth_priv *priv = (struct ldpaa_eth_priv *)net_dev->priv;
520 #endif
521         struct dpmac_link_state dpmac_link_state = { 0 };
522 #ifdef DEBUG
523         struct dpni_link_state link_state;
524 #endif
525         int err = 0;
526         struct dpni_queue d_queue;
527
528 #ifdef CONFIG_DM_ETH
529         if (eth_is_active(dev))
530                 return 0;
531 #else
532         if (net_dev->state == ETH_STATE_ACTIVE)
533                 return 0;
534 #endif
535
536         if (get_mc_boot_status() != 0) {
537                 printf("ERROR (MC is not booted)\n");
538                 return -ENODEV;
539         }
540
541         if (get_dpl_apply_status() == 0) {
542                 printf("ERROR (DPL is deployed. No device available)\n");
543                 return -ENODEV;
544         }
545
546         /* DPMAC initialization */
547         err = ldpaa_dpmac_setup(priv);
548         if (err < 0)
549                 goto err_dpmac_setup;
550
551         err = ldpaa_get_dpmac_state(priv, &dpmac_link_state);
552         if (err < 0)
553                 goto err_dpmac_bind;
554
555         /* DPMAC binding DPNI */
556         err = ldpaa_dpmac_bind(priv);
557         if (err)
558                 goto err_dpmac_bind;
559
560         /* DPNI initialization */
561         err = ldpaa_dpni_setup(priv);
562         if (err < 0)
563                 goto err_dpni_setup;
564
565         err = ldpaa_dpbp_setup();
566         if (err < 0)
567                 goto err_dpbp_setup;
568
569         /* DPNI binding DPBP */
570         err = ldpaa_dpni_bind(priv);
571         if (err)
572                 goto err_dpni_bind;
573
574 #ifdef CONFIG_DM_ETH
575         err = dpni_add_mac_addr(dflt_mc_io, MC_CMD_NO_FLAGS,
576                                 dflt_dpni->dpni_handle, plat->enetaddr);
577 #else
578         err = dpni_add_mac_addr(dflt_mc_io, MC_CMD_NO_FLAGS,
579                                 dflt_dpni->dpni_handle, net_dev->enetaddr);
580 #endif
581         if (err) {
582                 printf("dpni_add_mac_addr() failed\n");
583                 return err;
584         }
585
586         err = dpni_enable(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle);
587         if (err < 0) {
588                 printf("dpni_enable() failed\n");
589                 return err;
590         }
591
592         err = dpmac_set_link_state(dflt_mc_io, MC_CMD_NO_FLAGS,
593                                   priv->dpmac_handle, &dpmac_link_state);
594         if (err < 0) {
595                 printf("dpmac_set_link_state() failed\n");
596                 return err;
597         }
598
599 #ifdef DEBUG
600         printf("DPMAC link status: %d - ", dpmac_link_state.up);
601         dpmac_link_state.up == 0 ? printf("down\n") :
602         dpmac_link_state.up == 1 ? printf("up\n") : printf("error state\n");
603
604         err = dpni_get_link_state(dflt_mc_io, MC_CMD_NO_FLAGS,
605                                   dflt_dpni->dpni_handle, &link_state);
606         if (err < 0) {
607                 printf("dpni_get_link_state() failed\n");
608                 return err;
609         }
610
611         printf("DPNI link status: %d - ", link_state.up);
612         link_state.up == 0 ? printf("down\n") :
613         link_state.up == 1 ? printf("up\n") : printf("error state\n");
614 #endif
615
616         memset(&d_queue, 0, sizeof(struct dpni_queue));
617         err = dpni_get_queue(dflt_mc_io, MC_CMD_NO_FLAGS,
618                              dflt_dpni->dpni_handle, DPNI_QUEUE_RX,
619                              0, 0, &d_queue);
620         if (err) {
621                 printf("dpni_get_queue failed\n");
622                 goto err_get_queue;
623         }
624
625         priv->rx_dflt_fqid = d_queue.fqid;
626
627         err = dpni_get_qdid(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle,
628                             &priv->tx_qdid);
629         if (err) {
630                 printf("dpni_get_qdid() failed\n");
631                 goto err_qdid;
632         }
633
634         return dpmac_link_state.up;
635
636 err_qdid:
637 err_get_queue:
638         dpni_disable(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle);
639 err_dpni_bind:
640         ldpaa_dpbp_free();
641 err_dpbp_setup:
642         dpni_close(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle);
643 err_dpni_setup:
644 err_dpmac_bind:
645         dpmac_close(dflt_mc_io, MC_CMD_NO_FLAGS, priv->dpmac_handle);
646         dpmac_destroy(dflt_mc_io,
647                       dflt_dprc_handle,
648                       MC_CMD_NO_FLAGS, priv->dpmac_id);
649 err_dpmac_setup:
650         return err;
651 }
652
653 #ifdef CONFIG_DM_ETH
654 static void ldpaa_eth_stop(struct udevice *dev)
655 {
656         struct ldpaa_eth_priv *priv = dev_get_priv(dev);
657 #else
658 static void ldpaa_eth_stop(struct eth_device *net_dev)
659 {
660         struct ldpaa_eth_priv *priv = (struct ldpaa_eth_priv *)net_dev->priv;
661 #endif
662         struct phy_device *phydev = NULL;
663         int err = 0;
664
665 #ifdef CONFIG_DM_ETH
666         if (!eth_is_active(dev))
667                 return;
668 #else
669         if ((net_dev->state == ETH_STATE_PASSIVE) ||
670             (net_dev->state == ETH_STATE_INIT))
671                 return;
672 #endif
673
674 #ifdef DEBUG
675         ldpaa_eth_get_dpni_counter();
676 #ifdef CONFIG_DM_ETH
677         ldpaa_eth_get_dpmac_counter(dev);
678 #else
679         ldpaa_eth_get_dpmac_counter(net_dev);
680 #endif
681 #endif
682
683         err = dprc_disconnect(dflt_mc_io, MC_CMD_NO_FLAGS,
684                               dflt_dprc_handle, &dpmac_endpoint);
685         if (err < 0)
686                 printf("dprc_disconnect() failed dpmac_endpoint\n");
687
688         err = dpmac_close(dflt_mc_io, MC_CMD_NO_FLAGS, priv->dpmac_handle);
689         if (err < 0)
690                 printf("dpmac_close() failed\n");
691
692         err = dpmac_destroy(dflt_mc_io,
693                             dflt_dprc_handle,
694                             MC_CMD_NO_FLAGS,
695                             priv->dpmac_id);
696         if (err < 0)
697                 printf("dpmac_destroy() failed\n");
698
699         /* Stop Tx and Rx traffic */
700         err = dpni_disable(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle);
701         if (err < 0)
702                 printf("dpni_disable() failed\n");
703
704         phydev = ldpaa_get_phydev(priv);
705         if (phydev)
706                 phy_shutdown(phydev);
707
708         /* Free DPBP handle and reset. */
709         ldpaa_dpbp_free();
710
711         dpni_reset(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle);
712         if (err < 0)
713                 printf("dpni_reset() failed\n");
714
715         dpni_close(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle);
716         if (err < 0)
717                 printf("dpni_close() failed\n");
718 }
719
720 static void ldpaa_dpbp_drain_cnt(int count)
721 {
722         uint64_t buf_array[7];
723         void *addr;
724         int ret, i;
725
726         BUG_ON(count > 7);
727
728         do {
729                 ret = qbman_swp_acquire(dflt_dpio->sw_portal,
730                                         dflt_dpbp->dpbp_attr.bpid,
731                                         buf_array, count);
732                 if (ret < 0) {
733                         printf("qbman_swp_acquire() failed\n");
734                         return;
735                 }
736                 for (i = 0; i < ret; i++) {
737                         addr = (void *)buf_array[i];
738                         debug("Free: buffer addr =0x%p\n", addr);
739                         free(addr);
740                 }
741         } while (ret);
742 }
743
744 static void ldpaa_dpbp_drain(void)
745 {
746         int i;
747         for (i = 0; i < LDPAA_ETH_NUM_BUFS; i += 7)
748                 ldpaa_dpbp_drain_cnt(7);
749 }
750
751 static int ldpaa_bp_add_7(uint16_t bpid)
752 {
753         uint64_t buf_array[7];
754         u8 *addr;
755         int i;
756         struct qbman_release_desc rd;
757
758         for (i = 0; i < 7; i++) {
759                 addr = memalign(LDPAA_ETH_BUF_ALIGN, LDPAA_ETH_RX_BUFFER_SIZE);
760                 if (!addr) {
761                         printf("addr allocation failed\n");
762                         goto err_alloc;
763                 }
764                 memset(addr, 0x00, LDPAA_ETH_RX_BUFFER_SIZE);
765                 flush_dcache_range((u64)addr,
766                                    (u64)(addr + LDPAA_ETH_RX_BUFFER_SIZE));
767
768                 buf_array[i] = (uint64_t)addr;
769                 debug("Release: buffer addr =0x%p\n", addr);
770         }
771
772 release_bufs:
773         /* In case the portal is busy, retry until successful.
774          * This function is guaranteed to succeed in a reasonable amount
775          * of time.
776          */
777
778         do {
779                 mdelay(1);
780                 qbman_release_desc_clear(&rd);
781                 qbman_release_desc_set_bpid(&rd, bpid);
782         } while (qbman_swp_release(dflt_dpio->sw_portal, &rd, buf_array, i));
783
784         return i;
785
786 err_alloc:
787         if (i)
788                 goto release_bufs;
789
790         return 0;
791 }
792
793 static int ldpaa_dpbp_seed(uint16_t bpid)
794 {
795         int i;
796         int count;
797
798         for (i = 0; i < LDPAA_ETH_NUM_BUFS; i += 7) {
799                 count = ldpaa_bp_add_7(bpid);
800                 if (count < 7)
801                         printf("Buffer Seed= %d\n", count);
802         }
803
804         return 0;
805 }
806
807 static int ldpaa_dpbp_setup(void)
808 {
809         int err;
810
811         err = dpbp_open(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpbp->dpbp_attr.id,
812                         &dflt_dpbp->dpbp_handle);
813         if (err) {
814                 printf("dpbp_open() failed\n");
815                 goto err_open;
816         }
817
818         err = dpbp_enable(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpbp->dpbp_handle);
819         if (err) {
820                 printf("dpbp_enable() failed\n");
821                 goto err_enable;
822         }
823
824         err = dpbp_get_attributes(dflt_mc_io, MC_CMD_NO_FLAGS,
825                                   dflt_dpbp->dpbp_handle,
826                                   &dflt_dpbp->dpbp_attr);
827         if (err) {
828                 printf("dpbp_get_attributes() failed\n");
829                 goto err_get_attr;
830         }
831
832         err = ldpaa_dpbp_seed(dflt_dpbp->dpbp_attr.bpid);
833
834         if (err) {
835                 printf("Buffer seeding failed for DPBP %d (bpid=%d)\n",
836                        dflt_dpbp->dpbp_attr.id, dflt_dpbp->dpbp_attr.bpid);
837                 goto err_seed;
838         }
839
840         return 0;
841
842 err_seed:
843 err_get_attr:
844         dpbp_disable(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpbp->dpbp_handle);
845 err_enable:
846         dpbp_close(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpbp->dpbp_handle);
847 err_open:
848         return err;
849 }
850
851 static void ldpaa_dpbp_free(void)
852 {
853         ldpaa_dpbp_drain();
854         dpbp_disable(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpbp->dpbp_handle);
855         dpbp_reset(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpbp->dpbp_handle);
856         dpbp_close(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpbp->dpbp_handle);
857 }
858
859 static int ldpaa_dpmac_version_check(struct fsl_mc_io *mc_io,
860                                      struct ldpaa_eth_priv *priv)
861 {
862         int error;
863         uint16_t major_ver, minor_ver;
864
865         error = dpmac_get_api_version(dflt_mc_io, 0,
866                                         &major_ver,
867                                         &minor_ver);
868         if ((major_ver < DPMAC_VER_MAJOR) ||
869             (major_ver == DPMAC_VER_MAJOR && minor_ver < DPMAC_VER_MINOR)) {
870                 printf("DPMAC version mismatch found %u.%u,",
871                        major_ver, minor_ver);
872                 printf("supported version is %u.%u\n",
873                        DPMAC_VER_MAJOR, DPMAC_VER_MINOR);
874                 return error;
875         }
876
877         return error;
878 }
879
880 static int ldpaa_dpmac_setup(struct ldpaa_eth_priv *priv)
881 {
882         int err = 0;
883         struct dpmac_cfg dpmac_cfg;
884
885         dpmac_cfg.mac_id = priv->dpmac_id;
886
887         err = dpmac_create(dflt_mc_io,
888                            dflt_dprc_handle,
889                            MC_CMD_NO_FLAGS, &dpmac_cfg,
890                            &priv->dpmac_id);
891         if (err)
892                 printf("dpmac_create() failed\n");
893
894         err = ldpaa_dpmac_version_check(dflt_mc_io, priv);
895         if (err < 0) {
896                 printf("ldpaa_dpmac_version_check() failed: %d\n", err);
897                 goto err_version_check;
898         }
899
900         err = dpmac_open(dflt_mc_io,
901                          MC_CMD_NO_FLAGS,
902                          priv->dpmac_id,
903                          &priv->dpmac_handle);
904         if (err < 0) {
905                 printf("dpmac_open() failed: %d\n", err);
906                 goto err_open;
907         }
908
909         return err;
910
911 err_open:
912 err_version_check:
913         dpmac_destroy(dflt_mc_io,
914                       dflt_dprc_handle,
915                       MC_CMD_NO_FLAGS, priv->dpmac_id);
916
917         return err;
918 }
919
920 static int ldpaa_dpmac_bind(struct ldpaa_eth_priv *priv)
921 {
922         int err = 0;
923         struct dprc_connection_cfg dprc_connection_cfg = {
924                 /* If both rates are zero the connection */
925                 /* will be configured in "best effort" mode. */
926                 .committed_rate = 0,
927                 .max_rate = 0
928         };
929
930 #ifdef DEBUG
931         struct dprc_endpoint dbg_endpoint;
932         int state = 0;
933 #endif
934
935         memset(&dpmac_endpoint, 0, sizeof(struct dprc_endpoint));
936         strcpy(dpmac_endpoint.type, "dpmac");
937         dpmac_endpoint.id = priv->dpmac_id;
938
939         memset(&dpni_endpoint, 0, sizeof(struct dprc_endpoint));
940         strcpy(dpni_endpoint.type, "dpni");
941         dpni_endpoint.id = dflt_dpni->dpni_id;
942
943         err = dprc_connect(dflt_mc_io, MC_CMD_NO_FLAGS,
944                              dflt_dprc_handle,
945                              &dpmac_endpoint,
946                              &dpni_endpoint,
947                              &dprc_connection_cfg);
948         if (err)
949                 printf("dprc_connect() failed\n");
950
951 #ifdef DEBUG
952         err = dprc_get_connection(dflt_mc_io, MC_CMD_NO_FLAGS,
953                                     dflt_dprc_handle, &dpni_endpoint,
954                                     &dbg_endpoint, &state);
955         printf("%s, DPMAC Type= %s\n", __func__, dbg_endpoint.type);
956         printf("%s, DPMAC ID= %d\n", __func__, dbg_endpoint.id);
957         printf("%s, DPMAC State= %d\n", __func__, state);
958
959         memset(&dbg_endpoint, 0, sizeof(struct dprc_endpoint));
960         err = dprc_get_connection(dflt_mc_io, MC_CMD_NO_FLAGS,
961                                     dflt_dprc_handle, &dpmac_endpoint,
962                                     &dbg_endpoint, &state);
963         printf("%s, DPNI Type= %s\n", __func__, dbg_endpoint.type);
964         printf("%s, DPNI ID= %d\n", __func__, dbg_endpoint.id);
965         printf("%s, DPNI State= %d\n", __func__, state);
966 #endif
967         return err;
968 }
969
970 static int ldpaa_dpni_setup(struct ldpaa_eth_priv *priv)
971 {
972         int err;
973
974         /* and get a handle for the DPNI this interface is associate with */
975         err = dpni_open(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_id,
976                         &dflt_dpni->dpni_handle);
977         if (err) {
978                 printf("dpni_open() failed\n");
979                 goto err_open;
980         }
981         err = dpni_get_attributes(dflt_mc_io, MC_CMD_NO_FLAGS,
982                                   dflt_dpni->dpni_handle,
983                                   &dflt_dpni->dpni_attrs);
984         if (err) {
985                 printf("dpni_get_attributes() failed (err=%d)\n", err);
986                 goto err_get_attr;
987         }
988
989         /* Configure our buffers' layout */
990         dflt_dpni->buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT |
991                                    DPNI_BUF_LAYOUT_OPT_FRAME_STATUS |
992                                    DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE |
993                                    DPNI_BUF_LAYOUT_OPT_DATA_ALIGN;
994         dflt_dpni->buf_layout.pass_parser_result = true;
995         dflt_dpni->buf_layout.pass_frame_status = true;
996         dflt_dpni->buf_layout.private_data_size = LDPAA_ETH_SWA_SIZE;
997         /* HW erratum mandates data alignment in multiples of 256 */
998         dflt_dpni->buf_layout.data_align = LDPAA_ETH_BUF_ALIGN;
999
1000         /* ...rx, ... */
1001         err = dpni_set_buffer_layout(dflt_mc_io, MC_CMD_NO_FLAGS,
1002                                      dflt_dpni->dpni_handle,
1003                                      &dflt_dpni->buf_layout, DPNI_QUEUE_RX);
1004         if (err) {
1005                 printf("dpni_set_buffer_layout() failed");
1006                 goto err_buf_layout;
1007         }
1008
1009         /* ... tx, ... */
1010         /* remove Rx-only options */
1011         dflt_dpni->buf_layout.options &= ~(DPNI_BUF_LAYOUT_OPT_DATA_ALIGN |
1012                                       DPNI_BUF_LAYOUT_OPT_PARSER_RESULT);
1013         err = dpni_set_buffer_layout(dflt_mc_io, MC_CMD_NO_FLAGS,
1014                                      dflt_dpni->dpni_handle,
1015                                      &dflt_dpni->buf_layout, DPNI_QUEUE_TX);
1016         if (err) {
1017                 printf("dpni_set_buffer_layout() failed");
1018                 goto err_buf_layout;
1019         }
1020
1021         /* ... tx-confirm. */
1022         dflt_dpni->buf_layout.options &= ~DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE;
1023         err = dpni_set_buffer_layout(dflt_mc_io, MC_CMD_NO_FLAGS,
1024                                      dflt_dpni->dpni_handle,
1025                                      &dflt_dpni->buf_layout,
1026                                      DPNI_QUEUE_TX_CONFIRM);
1027         if (err) {
1028                 printf("dpni_set_buffer_layout() failed");
1029                 goto err_buf_layout;
1030         }
1031
1032         /* Now that we've set our tx buffer layout, retrieve the minimum
1033          * required tx data offset.
1034          */
1035         err = dpni_get_tx_data_offset(dflt_mc_io, MC_CMD_NO_FLAGS,
1036                                       dflt_dpni->dpni_handle,
1037                                       &priv->tx_data_offset);
1038         if (err) {
1039                 printf("dpni_get_tx_data_offset() failed\n");
1040                 goto err_data_offset;
1041         }
1042
1043         /* Warn in case TX data offset is not multiple of 64 bytes. */
1044         WARN_ON(priv->tx_data_offset % 64);
1045
1046         /* Accomodate SWA space. */
1047         priv->tx_data_offset += LDPAA_ETH_SWA_SIZE;
1048         debug("priv->tx_data_offset=%d\n", priv->tx_data_offset);
1049
1050         return 0;
1051
1052 err_data_offset:
1053 err_buf_layout:
1054 err_get_attr:
1055         dpni_close(dflt_mc_io, MC_CMD_NO_FLAGS, dflt_dpni->dpni_handle);
1056 err_open:
1057         return err;
1058 }
1059
1060 static int ldpaa_dpni_bind(struct ldpaa_eth_priv *priv)
1061 {
1062         struct dpni_pools_cfg pools_params;
1063         struct dpni_queue tx_queue;
1064         int err = 0;
1065
1066         memset(&pools_params, 0, sizeof(pools_params));
1067         pools_params.num_dpbp = 1;
1068         pools_params.pools[0].dpbp_id = (uint16_t)dflt_dpbp->dpbp_attr.id;
1069         pools_params.pools[0].buffer_size = LDPAA_ETH_RX_BUFFER_SIZE;
1070         err = dpni_set_pools(dflt_mc_io, MC_CMD_NO_FLAGS,
1071                              dflt_dpni->dpni_handle, &pools_params);
1072         if (err) {
1073                 printf("dpni_set_pools() failed\n");
1074                 return err;
1075         }
1076
1077         memset(&tx_queue, 0, sizeof(struct dpni_queue));
1078
1079         err = dpni_set_queue(dflt_mc_io, MC_CMD_NO_FLAGS,
1080                              dflt_dpni->dpni_handle,
1081                              DPNI_QUEUE_TX, 0, 0, &tx_queue);
1082
1083         if (err) {
1084                 printf("dpni_set_queue() failed\n");
1085                 return err;
1086         }
1087
1088         err = dpni_set_tx_confirmation_mode(dflt_mc_io, MC_CMD_NO_FLAGS,
1089                                             dflt_dpni->dpni_handle,
1090                                             DPNI_CONF_DISABLE);
1091         if (err) {
1092                 printf("dpni_set_tx_confirmation_mode() failed\n");
1093                 return err;
1094         }
1095
1096         return 0;
1097 }
1098
1099 #ifdef CONFIG_DM_ETH
1100 static int ldpaa_eth_probe(struct udevice *dev)
1101 {
1102         struct ofnode_phandle_args phandle;
1103
1104         /* Nothing to do if there is no "phy-handle" in the DTS node */
1105         if (dev_read_phandle_with_args(dev, "phy-handle", NULL,
1106                                        0, 0, &phandle)) {
1107                 return 0;
1108         }
1109
1110         init_phy(dev);
1111
1112         return 0;
1113 }
1114
1115 static uint32_t ldpaa_eth_get_dpmac_id(struct udevice *dev)
1116 {
1117         int port_node = dev_of_offset(dev);
1118
1119         return fdtdec_get_uint(gd->fdt_blob, port_node, "reg", -1);
1120 }
1121
1122 static const char *ldpaa_eth_get_phy_mode_str(struct udevice *dev)
1123 {
1124         int port_node = dev_of_offset(dev);
1125         const char *phy_mode_str;
1126
1127         phy_mode_str = fdt_getprop(gd->fdt_blob, port_node,
1128                                    "phy-connection-type", NULL);
1129         if (phy_mode_str)
1130                 return phy_mode_str;
1131
1132         phy_mode_str = fdt_getprop(gd->fdt_blob, port_node, "phy-mode", NULL);
1133         return phy_mode_str;
1134 }
1135
1136 static int ldpaa_eth_bind(struct udevice *dev)
1137 {
1138         const char *phy_mode_str = NULL;
1139         uint32_t dpmac_id;
1140         char eth_name[16];
1141         int phy_mode = -1;
1142
1143         phy_mode_str = ldpaa_eth_get_phy_mode_str(dev);
1144         if (phy_mode_str)
1145                 phy_mode = phy_get_interface_by_name(phy_mode_str);
1146         if (phy_mode == -1) {
1147                 dev_err(dev, "incorrect phy mode\n");
1148                 return -EINVAL;
1149         }
1150
1151         dpmac_id = ldpaa_eth_get_dpmac_id(dev);
1152         if (dpmac_id == -1) {
1153                 dev_err(dev, "missing reg field from the dpmac node\n");
1154                 return -EINVAL;
1155         }
1156
1157         sprintf(eth_name, "DPMAC%d@%s", dpmac_id, phy_mode_str);
1158         device_set_name(dev, eth_name);
1159
1160         return 0;
1161 }
1162
1163 static int ldpaa_eth_ofdata_to_platdata(struct udevice *dev)
1164 {
1165         struct ldpaa_eth_priv *priv = dev_get_priv(dev);
1166         const char *phy_mode_str;
1167
1168         priv->dpmac_id = ldpaa_eth_get_dpmac_id(dev);
1169         phy_mode_str = ldpaa_eth_get_phy_mode_str(dev);
1170         priv->phy_mode = phy_get_interface_by_name(phy_mode_str);
1171
1172         return 0;
1173 }
1174
1175 static const struct eth_ops ldpaa_eth_ops = {
1176         .start  = ldpaa_eth_open,
1177         .send   = ldpaa_eth_tx,
1178         .recv   = ldpaa_eth_pull_dequeue_rx,
1179         .stop   = ldpaa_eth_stop,
1180 };
1181
1182 static const struct udevice_id ldpaa_eth_of_ids[] = {
1183         { .compatible = "fsl,qoriq-mc-dpmac" },
1184 };
1185
1186 U_BOOT_DRIVER(ldpaa_eth) = {
1187         .name = "ldpaa_eth",
1188         .id = UCLASS_ETH,
1189         .of_match = ldpaa_eth_of_ids,
1190         .ofdata_to_platdata = ldpaa_eth_ofdata_to_platdata,
1191         .bind = ldpaa_eth_bind,
1192         .probe = ldpaa_eth_probe,
1193         .ops = &ldpaa_eth_ops,
1194         .priv_auto_alloc_size = sizeof(struct ldpaa_eth_priv),
1195         .platdata_auto_alloc_size = sizeof(struct eth_pdata),
1196 };
1197
1198 #else
1199
1200 static int ldpaa_eth_netdev_init(struct eth_device *net_dev,
1201                                  phy_interface_t enet_if)
1202 {
1203         int err;
1204         struct ldpaa_eth_priv *priv = (struct ldpaa_eth_priv *)net_dev->priv;
1205
1206         snprintf(net_dev->name, ETH_NAME_LEN, "DPMAC%d@%s", priv->dpmac_id,
1207                  phy_interface_strings[enet_if]);
1208
1209         net_dev->iobase = 0;
1210         net_dev->init = ldpaa_eth_open;
1211         net_dev->halt = ldpaa_eth_stop;
1212         net_dev->send = ldpaa_eth_tx;
1213         net_dev->recv = ldpaa_eth_pull_dequeue_rx;
1214
1215 #ifdef CONFIG_PHYLIB
1216         err = init_phy(net_dev);
1217         if (err < 0)
1218                 return err;
1219 #endif
1220
1221         err = eth_register(net_dev);
1222         if (err < 0) {
1223                 printf("eth_register() = %d\n", err);
1224                 return err;
1225         }
1226
1227         return 0;
1228 }
1229
1230 int ldpaa_eth_init(int dpmac_id, phy_interface_t enet_if)
1231 {
1232         struct eth_device               *net_dev = NULL;
1233         struct ldpaa_eth_priv           *priv = NULL;
1234         int                             err = 0;
1235
1236         /* Net device */
1237         net_dev = (struct eth_device *)malloc(sizeof(struct eth_device));
1238         if (!net_dev) {
1239                 printf("eth_device malloc() failed\n");
1240                 return -ENOMEM;
1241         }
1242         memset(net_dev, 0, sizeof(struct eth_device));
1243
1244         /* alloc the ldpaa ethernet private struct */
1245         priv = (struct ldpaa_eth_priv *)malloc(sizeof(struct ldpaa_eth_priv));
1246         if (!priv) {
1247                 printf("ldpaa_eth_priv malloc() failed\n");
1248                 free(net_dev);
1249                 return -ENOMEM;
1250         }
1251         memset(priv, 0, sizeof(struct ldpaa_eth_priv));
1252
1253         net_dev->priv = (void *)priv;
1254         priv->net_dev = (struct eth_device *)net_dev;
1255         priv->dpmac_id = dpmac_id;
1256         debug("%s dpmac_id=%d\n", __func__, dpmac_id);
1257
1258         err = ldpaa_eth_netdev_init(net_dev, enet_if);
1259         if (err)
1260                 goto err_netdev_init;
1261
1262         debug("ldpaa ethernet: Probed interface %s\n", net_dev->name);
1263         return 0;
1264
1265 err_netdev_init:
1266         free(priv);
1267         net_dev->priv = NULL;
1268         free(net_dev);
1269
1270         return err;
1271 }
1272 #endif