From: Doug Ledford <dledford@redhat.com>
Date: Mon, 1 Sep 2008 22:00:15 -0400
Subject: [openib] OFED-1.3.2-pre update
Message-id: 1220320815.27033.108.camel@firewall.xsintricity.com
O-Subject: [Patch RHEL5.3] OFED-1.3.2-pre update BZ439565
Bugzilla: 439565 443476 453110 458886 459052 458375 459052 230035 460623
This updates RHEL5 to the OFED-1.3.2-pre release. It's worth noting
that the OFED 1.3.2-pre release isn't really pre-release software, at
least not as in "we're working on it and it's not ready to be called a
release yet". Instead, it's that there have only been a few bugs fixed
in the 1.3 branch since the 1.3.1 release, and there weren't enough
serious bugs to cause the OFED people to pursue a complete 1.3.2 release
cycle. So, the current 1.3.2 tarballs are basically just 1.3.1 with a
few more bugs fixed than in 1.3.1.
The current 1.3.2 daily tarball I'm using is date 20080728-0355 and was
the last tarball they built. Presumably, nothing additional has been
reported against 1.3.1 for them to need to make another 1.3.2 tarball.
I toyed with the idea of doing a 1.4-pre patch, but the OFED 1.4 release
cycle *just* hit their first beta about a week ago, and it really isn't
ready for prime time yet. In addition, some of the new features in 1.4
are very invasive and if we made any attempt to include them at all, I
would certainly need more than one week to get it integrated in our
kernel.
This update is also relatively small compared to previous updates. No
where near the 96,000 line behemoth the last update was. Instead, this
is about 6,000 lines total. Much more sane.
It's been tested using ipath, cxgb3, and mthca hardware (and later this
week I'll add another ipath card and mlx4 to the mix) and it's been
tested in both ipoib and rdma mode talking to a rhel5 peer and a rhel4
peer.
Patch inlined here, but also available from my git tree under the
ofed-1.3.2-pre branch, aka:
git://git.engineering.redhat.com/users/dledford/rhel5-kernel.git ofed-1.3.2-pre
OFED 1.3.2-pre update
Signed-off-by: Doug Ledford <dledford@redhat.com>
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 5381c80..485f81e 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -55,7 +55,7 @@ struct addr_req {
int status;
};
-static void process_req(struct work_struct *work);
+static void process_req(void *work);
static DEFINE_MUTEX(lock);
static LIST_HEAD(req_list);
@@ -214,7 +214,7 @@ out:
return ret;
}
-static void process_req(struct work_struct *work)
+static void process_req(void *work)
{
struct addr_req *req, *temp_req;
struct sockaddr_in *src_in, *dst_in;
diff --git a/drivers/infiniband/core/fmr_pool.c b/drivers/infiniband/core/fmr_pool.c
index e8d5f6b..bbfc713 100644
--- a/drivers/infiniband/core/fmr_pool.c
+++ b/drivers/infiniband/core/fmr_pool.c
@@ -397,8 +397,23 @@ EXPORT_SYMBOL(ib_destroy_fmr_pool);
*/
int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
{
- int serial = atomic_inc_return(&pool->req_ser);
+ int serial;
+ struct ib_pool_fmr *fmr, *next;
+
+ /*
+ * The free_list holds FMRs that may have been used
+ * but have not been remapped enough times to be dirty.
+ * Put them on the dirty list now so that the cleanup
+ * thread will reap them too.
+ */
+ spin_lock_irq(&pool->pool_lock);
+ list_for_each_entry_safe(fmr, next, &pool->free_list, list) {
+ if (fmr->remap_count > 0)
+ list_move(&fmr->list, &pool->dirty_list);
+ }
+ spin_unlock_irq(&pool->pool_lock);
+ serial = atomic_inc_return(&pool->req_ser);
wake_up_process(pool->thread);
if (wait_event_interruptible(pool->force_wait,
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 223b1aa..81c9195 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -839,6 +839,7 @@ static void cm_work_handler(struct work_struct *_work)
unsigned long flags;
int empty;
int ret = 0;
+ int destroy_id;
spin_lock_irqsave(&cm_id_priv->lock, flags);
empty = list_empty(&cm_id_priv->work_list);
@@ -857,9 +858,9 @@ static void cm_work_handler(struct work_struct *_work)
destroy_cm_id(&cm_id_priv->id);
}
BUG_ON(atomic_read(&cm_id_priv->refcount)==0);
+ destroy_id = test_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags);
if (iwcm_deref_id(cm_id_priv)) {
- if (test_bit(IWCM_F_CALLBACK_DESTROY,
- &cm_id_priv->flags)) {
+ if (destroy_id) {
BUG_ON(!list_empty(&cm_id_priv->work_list));
free_cm_id(cm_id_priv);
}
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index d079ebb..4402d03 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -747,7 +747,9 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
break;
case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED:
kmem_cache_free(ib_mad_cache, mad_priv);
- break;
+ kfree(local);
+ ret = 1;
+ goto out;
case IB_MAD_RESULT_SUCCESS:
/* Treat like an incoming receive MAD */
port_priv = ib_get_mad_port(mad_agent_priv->agent.device,
@@ -2282,8 +2284,6 @@ static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
/* Empty wait list to prevent receives from finding a request */
list_splice_init(&mad_agent_priv->wait_list, &cancel_list);
- /* Empty local completion list as well */
- list_splice_init(&mad_agent_priv->local_list, &cancel_list);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
/* Report all cancelled requests */
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index 8994118..a2b1c4a 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -525,7 +525,7 @@ static void update_sm_ah(struct work_struct *work)
{
struct ib_sa_port *port =
container_of(work, struct ib_sa_port, update_task);
- struct ib_sa_sm_ah *new_ah, *old_ah;
+ struct ib_sa_sm_ah *new_ah;
struct ib_port_attr port_attr;
struct ib_ah_attr ah_attr;
@@ -561,12 +561,9 @@ static void update_sm_ah(struct work_struct *work)
}
spin_lock_irq(&port->ah_lock);
- old_ah = port->sm_ah;
port->sm_ah = new_ah;
spin_unlock_irq(&port->ah_lock);
- if (old_ah)
- kref_put(&old_ah->ref, free_sm_ah);
}
static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event)
@@ -577,8 +574,17 @@ static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event
event->event == IB_EVENT_PKEY_CHANGE ||
event->event == IB_EVENT_SM_CHANGE ||
event->event == IB_EVENT_CLIENT_REREGISTER) {
- struct ib_sa_device *sa_dev;
- sa_dev = container_of(handler, typeof(*sa_dev), event_handler);
+ unsigned long flags;
+ struct ib_sa_device *sa_dev =
+ container_of(handler, typeof(*sa_dev), event_handler);
+ struct ib_sa_port *port =
+ &sa_dev->port[event->element.port_num - sa_dev->start_port];
+
+ spin_lock_irqsave(&port->ah_lock, flags);
+ if (port->sm_ah)
+ kref_put(&port->sm_ah->ref, free_sm_ah);
+ port->sm_ah = NULL;
+ spin_unlock_irqrestore(&port->ah_lock, flags);
schedule_work(&sa_dev->port[event->element.port_num -
sa_dev->start_port].update_task);
@@ -683,6 +689,10 @@ static int alloc_mad(struct ib_sa_query *query, gfp_t gfp_mask)
unsigned long flags;
spin_lock_irqsave(&query->port->ah_lock, flags);
+ if (!query->port->sm_ah) {
+ spin_unlock_irqrestore(&query->port->ah_lock, flags);
+ return -EAGAIN;
+ }
kref_get(&query->port->sm_ah->ref);
query->sm_ah = query->port->sm_ah;
spin_unlock_irqrestore(&query->port->ah_lock, flags);
diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c
index beb2a38..a0fa586 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_hal.c
+++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c
@@ -178,7 +178,7 @@ int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
setup.size = 1UL << cq->size_log2;
setup.credits = 65535;
setup.credit_thres = 1;
- if (rdev_p->t3cdev_p->type == T3B)
+ if (rdev_p->t3cdev_p->type != T3A)
setup.ovfl_mode = 0;
else
setup.ovfl_mode = 1;
@@ -358,9 +358,10 @@ static void insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
cq->sw_wptr++;
}
-void cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
+int cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
{
u32 ptr;
+ int flushed = 0;
PDBG("%s wq %p cq %p\n", __FUNCTION__, wq, cq);
@@ -368,8 +369,11 @@ void cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
PDBG("%s rq_rptr %u rq_wptr %u skip count %u\n", __FUNCTION__,
wq->rq_rptr, wq->rq_wptr, count);
ptr = wq->rq_rptr + count;
- while (ptr++ != wq->rq_wptr)
+ while (ptr++ != wq->rq_wptr) {
insert_recv_cqe(wq, cq);
+ flushed++;
+ }
+ return flushed;
}
static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
@@ -393,18 +397,21 @@ static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
cq->sw_wptr++;
}
-void cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
+int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
{
__u32 ptr;
+ int flushed = 0;
struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2);
ptr = wq->sq_rptr + count;
- sqp += count;
+ sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
while (ptr != wq->sq_wptr) {
insert_sq_cqe(wq, cq, sqp);
- sqp++;
ptr++;
+ sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
+ flushed++;
}
+ return flushed;
}
/*
@@ -455,7 +462,8 @@ void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
ptr = cq->sw_rptr;
while (!Q_EMPTY(ptr, cq->sw_wptr)) {
cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
- if ((SQ_TYPE(*cqe) || (CQE_OPCODE(*cqe) == T3_READ_RESP)) &&
+ if ((SQ_TYPE(*cqe) ||
+ ((CQE_OPCODE(*cqe) == T3_READ_RESP) && wq->oldest_read)) &&
(CQE_QPID(*cqe) == wq->qpid))
(*count)++;
ptr++;
@@ -579,7 +587,7 @@ static int cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)
* caller aquires the ctrl_qp lock before the call
*/
static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
- u32 len, void *data, int completion)
+ u32 len, void *data)
{
u32 i, nr_wqe, copy_len;
u8 *copy_data;
@@ -615,7 +623,7 @@ static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
flag = 0;
if (i == (nr_wqe - 1)) {
/* last WQE */
- flag = completion ? T3_COMPLETION_FLAG : 0;
+ flag = T3_COMPLETION_FLAG;
if (len % 32)
utx_len = len / 32 + 1;
else
@@ -674,21 +682,20 @@ static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
return 0;
}
-/* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl, and pbl_size
- * OUT: stag index, actual pbl_size, pbl_addr allocated.
+/* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl_size and pbl_addr
+ * OUT: stag index
* TBD: shared memory region support
*/
static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
u32 *stag, u8 stag_state, u32 pdid,
enum tpt_mem_type type, enum tpt_mem_perm perm,
- u32 zbva, u64 to, u32 len, u8 page_size, __be64 *pbl,
- u32 *pbl_size, u32 *pbl_addr)
+ u32 zbva, u64 to, u32 len, u8 page_size,
+ u32 pbl_size, u32 pbl_addr)
{
int err;
struct tpt_entry tpt;
u32 stag_idx;
u32 wptr;
- int rereg = (*stag != T3_STAG_UNSET);
stag_state = stag_state > 0;
stag_idx = (*stag) >> 8;
@@ -702,30 +709,8 @@ static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
__FUNCTION__, stag_state, type, pdid, stag_idx);
- if (reset_tpt_entry)
- cxio_hal_pblpool_free(rdev_p, *pbl_addr, *pbl_size << 3);
- else if (!rereg) {
- *pbl_addr = cxio_hal_pblpool_alloc(rdev_p, *pbl_size << 3);
- if (!*pbl_addr) {
- return -ENOMEM;
- }
- }
-
mutex_lock(&rdev_p->ctrl_qp.lock);
- /* write PBL first if any - update pbl only if pbl list exist */
- if (pbl) {
-
- PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",
- __FUNCTION__, *pbl_addr, rdev_p->rnic_info.pbl_base,
- *pbl_size);
- err = cxio_hal_ctrl_qp_write_mem(rdev_p,
- (*pbl_addr >> 5),
- (*pbl_size << 3), pbl, 0);
- if (err)
- goto ret;
- }
-
/* write TPT entry */
if (reset_tpt_entry)
memset(&tpt, 0, sizeof(tpt));
@@ -740,23 +725,23 @@ static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |
V_TPT_PAGE_SIZE(page_size));
tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 :
- cpu_to_be32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, *pbl_addr)>>3));
+ cpu_to_be32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, pbl_addr)>>3));
tpt.len = cpu_to_be32(len);
tpt.va_hi = cpu_to_be32((u32) (to >> 32));
tpt.va_low_or_fbo = cpu_to_be32((u32) (to & 0xFFFFFFFFULL));
tpt.rsvd_bind_cnt_or_pstag = 0;
tpt.rsvd_pbl_size = reset_tpt_entry ? 0 :
- cpu_to_be32(V_TPT_PBL_SIZE((*pbl_size) >> 2));
+ cpu_to_be32(V_TPT_PBL_SIZE(pbl_size >> 2));
}
err = cxio_hal_ctrl_qp_write_mem(rdev_p,
stag_idx +
(rdev_p->rnic_info.tpt_base >> 5),
- sizeof(tpt), &tpt, 1);
+ sizeof(tpt), &tpt);
/* release the stag index to free pool */
if (reset_tpt_entry)
cxio_hal_put_stag(rdev_p->rscp, stag_idx);
-ret:
+
wptr = rdev_p->ctrl_qp.wptr;
mutex_unlock(&rdev_p->ctrl_qp.lock);
if (!err)
@@ -767,44 +752,67 @@ ret:
return err;
}
+int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,
+ u32 pbl_addr, u32 pbl_size)
+{
+ u32 wptr;
+ int err;
+
+ PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",
+ __FUNCTION__, pbl_addr, rdev_p->rnic_info.pbl_base,
+ pbl_size);
+
+ mutex_lock(&rdev_p->ctrl_qp.lock);
+ err = cxio_hal_ctrl_qp_write_mem(rdev_p, pbl_addr >> 5, pbl_size << 3,
+ pbl);
+ wptr = rdev_p->ctrl_qp.wptr;
+ mutex_unlock(&rdev_p->ctrl_qp.lock);
+ if (err)
+ return err;
+
+ if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,
+ SEQ32_GE(rdev_p->ctrl_qp.rptr,
+ wptr)))
+ return -ERESTARTSYS;
+
+ return 0;
+}
+
int cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
- u8 page_size, __be64 *pbl, u32 *pbl_size,
- u32 *pbl_addr)
+ u8 page_size, u32 pbl_size, u32 pbl_addr)
{
*stag = T3_STAG_UNSET;
return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
- zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
+ zbva, to, len, page_size, pbl_size, pbl_addr);
}
int cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
- u8 page_size, __be64 *pbl, u32 *pbl_size,
- u32 *pbl_addr)
+ u8 page_size, u32 pbl_size, u32 pbl_addr)
{
return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
- zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
+ zbva, to, len, page_size, pbl_size, pbl_addr);
}
int cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,
u32 pbl_addr)
{
- return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
- &pbl_size, &pbl_addr);
+ return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,
+ pbl_size, pbl_addr);
}
int cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)
{
- u32 pbl_size = 0;
*stag = T3_STAG_UNSET;
return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,
- NULL, &pbl_size, NULL);
+ 0, 0);
}
int cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
{
- return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
- NULL, NULL);
+ return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,
+ 0, 0);
}
int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
@@ -828,7 +836,8 @@ int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
wqe->mpaattrs = attr->mpaattrs;
wqe->qpcaps = attr->qpcaps;
wqe->ulpdu_size = cpu_to_be16(attr->tcp_emss);
- wqe->flags = cpu_to_be32(attr->flags);
+ wqe->rqe_count = cpu_to_be16(attr->rqe_count);
+ wqe->flags_rtr_type = cpu_to_be16(attr->flags|V_RTR_TYPE(attr->rtr_type));
wqe->ord = cpu_to_be32(attr->ord);
wqe->ird = cpu_to_be32(attr->ird);
wqe->qp_dma_addr = cpu_to_be64(attr->qp_dma_addr);
@@ -1134,6 +1143,18 @@ int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {
/*
+ * If this is an unsolicited read response, then the read
+ * was generated by the kernel driver as part of peer-2-peer
+ * connection setup. So ignore the completion.
+ */
+ if (!wq->oldest_read) {
+ if (CQE_STATUS(*hw_cqe))
+ wq->error = 1;
+ ret = -1;
+ goto skip_cqe;
+ }
+
+ /*
* Don't write to the HWCQ, so create a new read req CQE
* in local memory.
*/
diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h
index 99543d6..6e128f6 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_hal.h
+++ b/drivers/infiniband/hw/cxgb3/cxio_hal.h
@@ -53,6 +53,7 @@
#define T3_MAX_PBL_SIZE 256
#define T3_MAX_RQ_SIZE 1024
#define T3_MAX_NUM_STAG (1<<15)
+#define T3_MAX_MR_SIZE 0x100000000ULL
#define T3_STAG_UNSET 0xffffffff
@@ -153,14 +154,14 @@ int cxio_create_qp(struct cxio_rdev *rdev, u32 kernel_domain, struct t3_wq *wq,
int cxio_destroy_qp(struct cxio_rdev *rdev, struct t3_wq *wq,
struct cxio_ucontext *uctx);
int cxio_peek_cq(struct t3_wq *wr, struct t3_cq *cq, int opcode);
+int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,
+ u32 pbl_addr, u32 pbl_size);
int cxio_register_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid,
enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
- u8 page_size, __be64 *pbl, u32 *pbl_size,
- u32 *pbl_addr);
+ u8 page_size, u32 pbl_size, u32 pbl_addr);
int cxio_reregister_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid,
enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
- u8 page_size, __be64 *pbl, u32 *pbl_size,
- u32 *pbl_addr);
+ u8 page_size, u32 pbl_size, u32 pbl_addr);
int cxio_dereg_mem(struct cxio_rdev *rdev, u32 stag, u32 pbl_size,
u32 pbl_addr);
int cxio_allocate_window(struct cxio_rdev *rdev, u32 * stag, u32 pdid);
@@ -172,8 +173,8 @@ u32 cxio_hal_get_pdid(struct cxio_hal_resource *rscp);
void cxio_hal_put_pdid(struct cxio_hal_resource *rscp, u32 pdid);
int __init cxio_hal_init(void);
void __exit cxio_hal_exit(void);
-void cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count);
-void cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count);
+int cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count);
+int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count);
void cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count);
void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count);
void cxio_flush_hw_cq(struct t3_cq *cq);
diff --git a/drivers/infiniband/hw/cxgb3/cxio_resource.c b/drivers/infiniband/hw/cxgb3/cxio_resource.c
index d3095ae..d0526f0 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_resource.c
+++ b/drivers/infiniband/hw/cxgb3/cxio_resource.c
@@ -250,7 +250,6 @@ void cxio_hal_destroy_resource(struct cxio_hal_resource *rscp)
*/
#define MIN_PBL_SHIFT 8 /* 256B == min PBL size (32 entries) */
-#define PBL_CHUNK 2*1024*1024
u32 cxio_hal_pblpool_alloc(struct cxio_rdev *rdev_p, int size)
{
@@ -267,14 +266,35 @@ void cxio_hal_pblpool_free(struct cxio_rdev *rdev_p, u32 addr, int size)
int cxio_hal_pblpool_create(struct cxio_rdev *rdev_p)
{
- unsigned long i;
+ unsigned pbl_start, pbl_chunk;
+
rdev_p->pbl_pool = gen_pool_create(MIN_PBL_SHIFT, -1);
- if (rdev_p->pbl_pool)
- for (i = rdev_p->rnic_info.pbl_base;
- i <= rdev_p->rnic_info.pbl_top - PBL_CHUNK + 1;
- i += PBL_CHUNK)
- gen_pool_add(rdev_p->pbl_pool, i, PBL_CHUNK, -1);
- return rdev_p->pbl_pool ? 0 : -ENOMEM;
+ if (!rdev_p->pbl_pool)
+ return -ENOMEM;
+
+ pbl_start = rdev_p->rnic_info.pbl_base;
+ pbl_chunk = rdev_p->rnic_info.pbl_top - pbl_start + 1;
+
+ while (pbl_start < rdev_p->rnic_info.pbl_top) {
+ pbl_chunk = min(rdev_p->rnic_info.pbl_top - pbl_start + 1,
+ pbl_chunk);
+ if (gen_pool_add(rdev_p->pbl_pool, pbl_start, pbl_chunk, -1)) {
+ PDBG("%s failed to add PBL chunk (%x/%x)\n",
+ __func__, pbl_start, pbl_chunk);
+ if (pbl_chunk <= 1024 << MIN_PBL_SHIFT) {
+ printk(KERN_WARNING MOD "%s: Failed to add all PBL chunks (%x/%x)\n",
+ __func__, pbl_start, rdev_p->rnic_info.pbl_top - pbl_start);
+ return 0;
+ }
+ pbl_chunk >>= 1;
+ } else {
+ PDBG("%s added PBL chunk (%x/%x)\n",
+ __func__, pbl_start, pbl_chunk);
+ pbl_start += pbl_chunk;
+ }
+ }
+
+ return 0;
}
void cxio_hal_pblpool_destroy(struct cxio_rdev *rdev_p)
diff --git a/drivers/infiniband/hw/cxgb3/cxio_wr.h b/drivers/infiniband/hw/cxgb3/cxio_wr.h
index c84d4ac..c1f0cdd 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_wr.h
+++ b/drivers/infiniband/hw/cxgb3/cxio_wr.h
@@ -278,6 +278,17 @@ enum t3_qp_caps {
uP_RI_QP_STAG0_ENABLE = 0x10
} __attribute__ ((packed));
+enum rdma_init_rtr_types {
+ RTR_READ = 1,
+ RTR_WRITE = 2,
+ RTR_SEND = 3,
+};
+
+#define S_RTR_TYPE 2
+#define M_RTR_TYPE 0x3
+#define V_RTR_TYPE(x) ((x) << S_RTR_TYPE)
+#define G_RTR_TYPE(x) ((((x) >> S_RTR_TYPE)) & M_RTR_TYPE)
+
struct t3_rdma_init_attr {
u32 tid;
u32 qpid;
@@ -293,7 +304,9 @@ struct t3_rdma_init_attr {
u32 ird;
u64 qp_dma_addr;
u32 qp_dma_size;
- u32 flags;
+ enum rdma_init_rtr_types rtr_type;
+ u16 flags;
+ u16 rqe_count;
u32 irs;
};
@@ -309,8 +322,8 @@ struct t3_rdma_init_wr {
u8 mpaattrs; /* 5 */
u8 qpcaps;
__be16 ulpdu_size;
- __be32 flags; /* bits 31-1 - reservered */
- /* bit 0 - set if RECV posted */
+ __be16 flags_rtr_type;
+ __be16 rqe_count;
__be32 ord; /* 6 */
__be32 ird;
__be64 qp_dma_addr; /* 7 */
@@ -324,7 +337,8 @@ struct t3_genbit {
};
enum rdma_init_wr_flags {
- RECVS_POSTED = 1,
+ MPA_INITIATOR = (1<<0),
+ PRIV_QP = (1<<1),
};
union t3_wr {
diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c
index 0315c9d..98a768f 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.c
+++ b/drivers/infiniband/hw/cxgb3/iwch.c
@@ -83,6 +83,7 @@ static void rnic_init(struct iwch_dev *rnicp)
rnicp->attr.max_phys_buf_entries = T3_MAX_PBL_SIZE;
rnicp->attr.max_pds = T3_MAX_NUM_PD - 1;
rnicp->attr.mem_pgsizes_bitmask = 0x7FFF; /* 4KB-128MB */
+ rnicp->attr.max_mr_size = T3_MAX_MR_SIZE;
rnicp->attr.can_resize_wq = 0;
rnicp->attr.max_rdma_reads_per_qp = 8;
rnicp->attr.max_rdma_read_resources =
diff --git a/drivers/infiniband/hw/cxgb3/iwch.h b/drivers/infiniband/hw/cxgb3/iwch.h
index caf4e60..238c103 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.h
+++ b/drivers/infiniband/hw/cxgb3/iwch.h
@@ -66,6 +66,7 @@ struct iwch_rnic_attributes {
* size (4k)^i. Phys block list mode unsupported.
*/
u32 mem_pgsizes_bitmask;
+ u64 max_mr_size;
u8 can_resize_wq;
/*
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c
index 6798f2c..0c26837 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cm.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c
@@ -65,10 +65,14 @@ static char *states[] = {
};
#endif
-static int ep_timeout_secs = 10;
+int peer2peer = 0;
+module_param(peer2peer, int, 0644);
+MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)");
+
+static int ep_timeout_secs = 60;
module_param(ep_timeout_secs, int, 0644);
MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout "
- "in seconds (default=10)");
+ "in seconds (default=60)");
static int mpa_rev = 1;
module_param(mpa_rev, int, 0644);
@@ -127,6 +131,12 @@ static void start_ep_timer(struct iwch_ep *ep)
static void stop_ep_timer(struct iwch_ep *ep)
{
PDBG("%s ep %p\n", __FUNCTION__, ep);
+ if (!timer_pending(&ep->timer)) {
+ printk(KERN_ERR "%s timer stopped when its not running! ep %p state %u\n",
+ __FUNCTION__, ep, ep->com.state);
+ WARN_ON(1);
+ return;
+ }
del_timer_sync(&ep->timer);
put_ep(&ep->com);
}
@@ -510,7 +520,7 @@ static void send_mpa_req(struct iwch_ep *ep, struct sk_buff *skb)
skb_reset_transport_header(skb);
len = skb->len;
req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
- req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
req->wr_lo = htonl(V_WR_TID(ep->hwtid));
req->len = htonl(len);
req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
@@ -561,7 +571,7 @@ static int send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)
set_arp_failure_handler(skb, arp_failure_discard);
skb_reset_transport_header(skb);
req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
- req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
req->wr_lo = htonl(V_WR_TID(ep->hwtid));
req->len = htonl(mpalen);
req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
@@ -613,7 +623,7 @@ static int send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
skb_reset_transport_header(skb);
len = skb->len;
req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
- req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
req->wr_lo = htonl(V_WR_TID(ep->hwtid));
req->len = htonl(len);
req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
@@ -881,6 +891,7 @@ static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb)
* the MPA header is valid.
*/
state_set(&ep->com, FPDU_MODE);
+ ep->mpa_attr.initiator = 1;
ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
ep->mpa_attr.recv_marker_enabled = markers_enabled;
ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
@@ -903,8 +914,14 @@ static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb)
/* bind QP and TID with INIT_WR */
err = iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, mask, &attrs, 1);
- if (!err)
- goto out;
+ if (err)
+ goto err;
+
+ if (peer2peer && iwch_rqes_posted(ep->com.qp) == 0) {
+ iwch_post_zb_read(ep->com.qp);
+ }
+
+ goto out;
err:
abort_connection(ep, skb, GFP_KERNEL);
out:
@@ -997,6 +1014,7 @@ static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb)
* If we get here we have accumulated the entire mpa
* start reply message including private data.
*/
+ ep->mpa_attr.initiator = 0;
ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
ep->mpa_attr.recv_marker_enabled = markers_enabled;
ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
@@ -1067,17 +1085,33 @@ static int tx_ack(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
PDBG("%s ep %p credits %u\n", __FUNCTION__, ep, credits);
- if (credits == 0)
+ if (credits == 0) {
+ PDBG(KERN_ERR "%s 0 credit ack ep %p state %u\n",
+ __FUNCTION__, ep, state_read(&ep->com));
return CPL_RET_BUF_DONE;
+ }
+
BUG_ON(credits != 1);
- BUG_ON(ep->mpa_skb == NULL);
- kfree_skb(ep->mpa_skb);
- ep->mpa_skb = NULL;
dst_confirm(ep->dst);
- if (state_read(&ep->com) == MPA_REP_SENT) {
- ep->com.rpl_done = 1;
- PDBG("waking up ep %p\n", ep);
- wake_up(&ep->com.waitq);
+ if (!ep->mpa_skb) {
+ PDBG("%s rdma_init wr_ack ep %p state %u\n",
+ __FUNCTION__, ep, state_read(&ep->com));
+ if (ep->mpa_attr.initiator) {
+ PDBG("%s initiator ep %p state %u\n",
+ __FUNCTION__, ep, state_read(&ep->com));
+ if (peer2peer)
+ iwch_post_zb_read(ep->com.qp);
+ } else {
+ PDBG("%s responder ep %p state %u\n",
+ __FUNCTION__, ep, state_read(&ep->com));
+ ep->com.rpl_done = 1;
+ wake_up(&ep->com.waitq);
+ }
+ } else {
+ PDBG("%s lsm ack ep %p state %u freeing skb\n",
+ __FUNCTION__, ep, state_read(&ep->com));
+ kfree_skb(ep->mpa_skb);
+ ep->mpa_skb = NULL;
}
return CPL_RET_BUF_DONE;
}
@@ -1085,8 +1119,11 @@ static int tx_ack(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
static int abort_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
{
struct iwch_ep *ep = ctx;
+ unsigned long flags;
+ int release = 0;
PDBG("%s ep %p\n", __FUNCTION__, ep);
+ BUG_ON(!ep);
/*
* We get 2 abort replies from the HW. The first one must
@@ -1097,9 +1134,22 @@ static int abort_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
return CPL_RET_BUF_DONE;
}
- close_complete_upcall(ep);
- state_set(&ep->com, DEAD);
- release_ep_resources(ep);
+ spin_lock_irqsave(&ep->com.lock, flags);
+ switch (ep->com.state) {
+ case ABORTING:
+ close_complete_upcall(ep);
+ __state_set(&ep->com, DEAD);
+ release = 1;
+ break;
+ default:
+ printk(KERN_ERR "%s ep %p state %d\n",
+ __FUNCTION__, ep, ep->com.state);
+ break;
+ }
+ spin_unlock_irqrestore(&ep->com.lock, flags);
+
+ if (release)
+ release_ep_resources(ep);
return CPL_RET_BUF_DONE;
}
@@ -1121,7 +1171,7 @@ static int act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
status2errno(rpl->status));
connect_reply_upcall(ep, status2errno(rpl->status));
state_set(&ep->com, DEAD);
- if (ep->com.tdev->type == T3B && act_open_has_tid(rpl->status))
+ if (ep->com.tdev->type != T3A && act_open_has_tid(rpl->status))
release_tid(ep->com.tdev, GET_TID(rpl), NULL);
cxgb3_free_atid(ep->com.tdev, ep->atid);
dst_release(ep->dst);
@@ -1252,7 +1302,7 @@ static void reject_cr(struct t3cdev *tdev, u32 hwtid, __be32 peer_ip,
skb_trim(skb, sizeof(struct cpl_tid_release));
skb_get(skb);
- if (tdev->type == T3B)
+ if (tdev->type != T3A)
release_tid(tdev, hwtid, skb);
else {
struct cpl_pass_accept_rpl *rpl;
@@ -1472,7 +1522,8 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct sk_buff *rpl_skb;
struct iwch_qp_attributes attrs;
int ret;
- int state;
+ int release = 0;
+ unsigned long flags;
if (is_neg_adv_abort(req->status)) {
PDBG("%s neg_adv_abort ep %p tid %d\n", __FUNCTION__, ep,
@@ -1490,9 +1541,9 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
return CPL_RET_BUF_DONE;
}
- state = state_read(&ep->com);
- PDBG("%s ep %p state %u\n", __FUNCTION__, ep, state);
- switch (state) {
+ spin_lock_irqsave(&ep->com.lock, flags);
+ PDBG("%s ep %p state %u\n", __FUNCTION__, ep, ep->com.state);
+ switch (ep->com.state) {
case CONNECTING:
break;
case MPA_REQ_WAIT:
@@ -1538,21 +1589,25 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
break;
case DEAD:
PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __FUNCTION__);
+ spin_unlock_irqrestore(&ep->com.lock, flags);
return CPL_RET_BUF_DONE;
default:
BUG_ON(1);
break;
}
dst_confirm(ep->dst);
+ if (ep->com.state != ABORTING) {
+ __state_set(&ep->com, DEAD);
+ release = 1;
+ }
+ spin_unlock_irqrestore(&ep->com.lock, flags);
rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL);
if (!rpl_skb) {
printk(KERN_ERR MOD "%s - cannot allocate skb!\n",
__FUNCTION__);
- dst_release(ep->dst);
- l2t_release(L2DATA(ep->com.tdev), ep->l2t);
- put_ep(&ep->com);
- return CPL_RET_BUF_DONE;
+ release = 1;
+ goto out;
}
rpl_skb->priority = CPL_PRIORITY_DATA;
rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl));
@@ -1561,10 +1616,9 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid));
rpl->cmd = CPL_ABORT_NO_RST;
cxgb3_ofld_send(ep->com.tdev, rpl_skb);
- if (state != ABORTING) {
- state_set(&ep->com, DEAD);
+out:
+ if (release)
release_ep_resources(ep);
- }
return CPL_RET_BUF_DONE;
}
@@ -1598,8 +1652,8 @@ static int close_con_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
release = 1;
break;
case ABORTING:
- break;
case DEAD:
+ break;
default:
BUG_ON(1);
break;
@@ -1663,15 +1717,18 @@ static void ep_timeout(unsigned long arg)
struct iwch_ep *ep = (struct iwch_ep *)arg;
struct iwch_qp_attributes attrs;
unsigned long flags;
+ int abort=1;
spin_lock_irqsave(&ep->com.lock, flags);
PDBG("%s ep %p tid %u state %d\n", __FUNCTION__, ep, ep->hwtid,
ep->com.state);
switch (ep->com.state) {
case MPA_REQ_SENT:
+ __state_set(&ep->com, ABORTING);
connect_reply_upcall(ep, -ETIMEDOUT);
break;
case MPA_REQ_WAIT:
+ __state_set(&ep->com, ABORTING);
break;
case CLOSING:
case MORIBUND:
@@ -1681,13 +1738,17 @@ static void ep_timeout(unsigned long arg)
ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
}
+ __state_set(&ep->com, ABORTING);
break;
default:
- BUG();
+ printk(KERN_ERR "%s unexpected state ep %p state %u\n",
+ __FUNCTION__, ep, ep->com.state);
+ WARN_ON(1);
+ abort=0;
}
- __state_set(&ep->com, CLOSING);
spin_unlock_irqrestore(&ep->com.lock, flags);
- abort_connection(ep, NULL, GFP_ATOMIC);
+ if (abort)
+ abort_connection(ep, NULL, GFP_ATOMIC);
put_ep(&ep->com);
}
@@ -1747,7 +1808,7 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
/* bind QP to EP and move to RTS */
attrs.mpa_attr = ep->mpa_attr;
- attrs.max_ird = ep->ord;
+ attrs.max_ird = ep->ird;
attrs.max_ord = ep->ord;
attrs.llp_stream_handle = ep;
attrs.next_state = IWCH_QP_STATE_RTS;
@@ -1764,16 +1825,19 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
if (err)
goto err;
+ /* if needed, wait for wr_ack */
+ if (iwch_rqes_posted(qp)) {
+ wait_event(ep->com.waitq, ep->com.rpl_done);
+ err = ep->com.rpl_err;
+ if (err)
+ goto err;
+ }
+
err = send_mpa_reply(ep, conn_param->private_data,
conn_param->private_data_len);
if (err)
goto err;
- /* wait for wr_ack */
- wait_event(ep->com.waitq, ep->com.rpl_done);
- err = ep->com.rpl_err;
- if (err)
- goto err;
state_set(&ep->com, FPDU_MODE);
established_upcall(ep);
@@ -1970,40 +2034,38 @@ int iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, gfp_t gfp)
PDBG("%s ep %p state %s, abrupt %d\n", __FUNCTION__, ep,
states[ep->com.state], abrupt);
- if (ep->com.state == DEAD) {
- PDBG("%s already dead ep %p\n", __FUNCTION__, ep);
- goto out;
- }
-
- if (abrupt) {
- if (ep->com.state != ABORTING) {
- ep->com.state = ABORTING;
- close = 1;
- }
- goto out;
- }
-
switch (ep->com.state) {
case MPA_REQ_WAIT:
case MPA_REQ_SENT:
case MPA_REQ_RCVD:
case MPA_REP_SENT:
case FPDU_MODE:
- start_ep_timer(ep);
- ep->com.state = CLOSING;
close = 1;
+ if (abrupt)
+ ep->com.state = ABORTING;
+ else {
+ ep->com.state = CLOSING;
+ start_ep_timer(ep);
+ }
break;
case CLOSING:
- ep->com.state = MORIBUND;
close = 1;
+ if (abrupt) {
+ stop_ep_timer(ep);
+ ep->com.state = ABORTING;
+ } else
+ ep->com.state = MORIBUND;
break;
case MORIBUND:
+ case ABORTING:
+ case DEAD:
+ PDBG("%s ignoring disconnect ep %p state %u\n",
+ __FUNCTION__, ep, ep->com.state);
break;
default:
BUG();
break;
}
-out:
spin_unlock_irqrestore(&ep->com.lock, flags);
if (close) {
if (abrupt)
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.h b/drivers/infiniband/hw/cxgb3/iwch_cm.h
index 6107e7c..c0978a8 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cm.h
+++ b/drivers/infiniband/hw/cxgb3/iwch_cm.h
@@ -56,6 +56,7 @@
#define put_ep(ep) { \
PDBG("put_ep (via %s:%u) ep %p refcnt %d\n", __FUNCTION__, __LINE__, \
ep, atomic_read(&((ep)->kref.refcount))); \
+ WARN_ON(atomic_read(&((ep)->kref.refcount)) < 1); \
kref_put(&((ep)->kref), __free_ep); \
}
@@ -225,5 +226,6 @@ int iwch_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, st
int __init iwch_cm_init(void);
void __exit iwch_cm_term(void);
+extern int peer2peer;
#endif /* _IWCH_CM_H_ */
diff --git a/drivers/infiniband/hw/cxgb3/iwch_mem.c b/drivers/infiniband/hw/cxgb3/iwch_mem.c
index a6c2c4b..2fd8939 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_mem.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_mem.c
@@ -35,17 +35,26 @@
#include <rdma/ib_verbs.h>
#include "cxio_hal.h"
+#include "cxio_resource.h"
#include "iwch.h"
#include "iwch_provider.h"
-int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
- struct iwch_mr *mhp,
- int shift,
- __be64 *page_list)
+static void iwch_finish_mem_reg(struct iwch_mr *mhp, u32 stag)
{
- u32 stag;
u32 mmid;
+ mhp->attr.state = 1;
+ mhp->attr.stag = stag;
+ mmid = stag >> 8;
+ mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
+ insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid);
+ PDBG("%s mmid 0x%x mhp %p\n", __FUNCTION__, mmid, mhp);
+}
+
+int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
+ struct iwch_mr *mhp, int shift)
+{
+ u32 stag;
if (cxio_register_phys_mem(&rhp->rdev,
&stag, mhp->attr.pdid,
@@ -53,28 +62,21 @@ int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
mhp->attr.zbva,
mhp->attr.va_fbo,
mhp->attr.len,
- shift-12,
- page_list,
- &mhp->attr.pbl_size, &mhp->attr.pbl_addr))
+ shift - 12,
+ mhp->attr.pbl_size, mhp->attr.pbl_addr))
return -ENOMEM;
- mhp->attr.state = 1;
- mhp->attr.stag = stag;
- mmid = stag >> 8;
- mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
- insert_handle(rhp, &rhp->mmidr, mhp, mmid);
- PDBG("%s mmid 0x%x mhp %p\n", __FUNCTION__, mmid, mhp);
+
+ iwch_finish_mem_reg(mhp, stag);
+
return 0;
}
int iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php,
struct iwch_mr *mhp,
int shift,
- __be64 *page_list,
int npages)
{
u32 stag;
- u32 mmid;
-
/* We could support this... */
if (npages > mhp->attr.pbl_size)
@@ -87,19 +89,40 @@ int iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php,
mhp->attr.zbva,
mhp->attr.va_fbo,
mhp->attr.len,
- shift-12,
- page_list,
- &mhp->attr.pbl_size, &mhp->attr.pbl_addr))
+ shift - 12,
+ mhp->attr.pbl_size, mhp->attr.pbl_addr))
return -ENOMEM;
- mhp->attr.state = 1;
- mhp->attr.stag = stag;
- mmid = stag >> 8;
- mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
- insert_handle(rhp, &rhp->mmidr, mhp, mmid);
- PDBG("%s mmid 0x%x mhp %p\n", __FUNCTION__, mmid, mhp);
+
+ iwch_finish_mem_reg(mhp, stag);
+
+ return 0;
+}
+
+int iwch_alloc_pbl(struct iwch_mr *mhp, int npages)
+{
+ mhp->attr.pbl_addr = cxio_hal_pblpool_alloc(&mhp->rhp->rdev,
+ npages << 3);
+
+ if (!mhp->attr.pbl_addr)
+ return -ENOMEM;
+
+ mhp->attr.pbl_size = npages;
+
return 0;
}
+void iwch_free_pbl(struct iwch_mr *mhp)
+{
+ cxio_hal_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
+ mhp->attr.pbl_size << 3);
+}
+
+int iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset)
+{
+ return cxio_write_pbl(&mhp->rhp->rdev, pages,
+ mhp->attr.pbl_addr + (offset << 3), npages);
+}
+
int build_phys_page_list(struct ib_phys_buf *buffer_list,
int num_phys_buf,
u64 *iova_start,
@@ -122,6 +145,13 @@ int build_phys_page_list(struct ib_phys_buf *buffer_list,
*total_size += buffer_list[i].size;
if (i > 0)
mask |= buffer_list[i].addr;
+ else
+ mask |= buffer_list[i].addr & PAGE_MASK;
+ if (i != num_phys_buf - 1)
+ mask |= buffer_list[i].addr + buffer_list[i].size;
+ else
+ mask |= (buffer_list[i].addr + buffer_list[i].size +
+ PAGE_SIZE - 1) & PAGE_MASK;
}
if (*total_size > 0xFFFFFFFFULL)
@@ -129,14 +159,8 @@ int build_phys_page_list(struct ib_phys_buf *buffer_list,
/* Find largest page shift we can use to cover buffers */
for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift))
- if (num_phys_buf > 1) {
- if ((1ULL << *shift) & mask)
- break;
- } else
- if (1ULL << *shift >=
- buffer_list[0].size +
- (buffer_list[0].addr & ((1ULL << *shift) - 1)))
- break;
+ if ((1ULL << *shift) & mask)
+ break;
buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1);
buffer_list[0].addr &= ~0ull << *shift;
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 69b1204..974473b 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -189,7 +189,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, int entries, int ve
return ERR_PTR(-ENOMEM);
}
chp->rhp = rhp;
- chp->ibcq.cqe = (1 << chp->cq.size_log2) - 1;
+ chp->ibcq.cqe = 1 << chp->cq.size_log2;
spin_lock_init(&chp->lock);
atomic_set(&chp->refcnt, 1);
init_waitqueue_head(&chp->wait);
@@ -442,6 +442,7 @@ static int iwch_dereg_mr(struct ib_mr *ib_mr)
mmid = mhp->attr.stag >> 8;
cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
mhp->attr.pbl_addr);
+ iwch_free_pbl(mhp);
remove_handle(rhp, &rhp->mmidr, mmid);
if (mhp->kva)
kfree((void *) (unsigned long) mhp->kva);
@@ -475,6 +476,8 @@ static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
if (!mhp)
return ERR_PTR(-ENOMEM);
+ mhp->rhp = rhp;
+
/* First check that we have enough alignment */
if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
ret = -EINVAL;
@@ -492,7 +495,17 @@ static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
if (ret)
goto err;
- mhp->rhp = rhp;
+ ret = iwch_alloc_pbl(mhp, npages);
+ if (ret) {
+ kfree(page_list);
+ goto err_pbl;
+ }
+
+ ret = iwch_write_pbl(mhp, page_list, npages, 0);
+ kfree(page_list);
+ if (ret)
+ goto err_pbl;
+
mhp->attr.pdid = php->pdid;
mhp->attr.zbva = 0;
@@ -502,12 +515,15 @@ static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
mhp->attr.len = (u32) total_size;
mhp->attr.pbl_size = npages;
- ret = iwch_register_mem(rhp, php, mhp, shift, page_list);
- kfree(page_list);
- if (ret) {
- goto err;
- }
+ ret = iwch_register_mem(rhp, php, mhp, shift);
+ if (ret)
+ goto err_pbl;
+
return &mhp->ibmr;
+
+err_pbl:
+ iwch_free_pbl(mhp);
+
err:
kfree(mhp);
return ERR_PTR(ret);
@@ -560,7 +576,7 @@ static int iwch_reregister_phys_mem(struct ib_mr *mr,
return ret;
}
- ret = iwch_reregister_mem(rhp, php, &mh, shift, page_list, npages);
+ ret = iwch_reregister_mem(rhp, php, &mh, shift, npages);
kfree(page_list);
if (ret) {
return ret;
@@ -602,6 +618,8 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
if (!mhp)
return ERR_PTR(-ENOMEM);
+ mhp->rhp = rhp;
+
mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc);
if (IS_ERR(mhp->umem)) {
err = PTR_ERR(mhp->umem);
@@ -615,10 +633,14 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
n += chunk->nents;
- pages = kmalloc(n * sizeof(u64), GFP_KERNEL);
+ err = iwch_alloc_pbl(mhp, n);
+ if (err)
+ goto err;
+
+ pages = (__be64 *) __get_free_page(GFP_KERNEL);
if (!pages) {
err = -ENOMEM;
- goto err;
+ goto err_pbl;
}
i = n = 0;
@@ -630,25 +652,38 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
pages[i++] = cpu_to_be64(sg_dma_address(
&chunk->page_list[j]) +
mhp->umem->page_size * k);
+ if (i == PAGE_SIZE / sizeof *pages) {
+ err = iwch_write_pbl(mhp, pages, i, n);
+ if (err)
+ goto pbl_done;
+ n += i;
+ i = 0;
+ }
}
}
- mhp->rhp = rhp;
+ if (i)
+ err = iwch_write_pbl(mhp, pages, i, n);
+
+pbl_done:
+ free_page((unsigned long) pages);
+ if (err)
+ goto err_pbl;
+
mhp->attr.pdid = php->pdid;
mhp->attr.zbva = 0;
mhp->attr.perms = iwch_ib_to_tpt_access(acc);
mhp->attr.va_fbo = virt;
mhp->attr.page_size = shift - 12;
mhp->attr.len = (u32) length;
- mhp->attr.pbl_size = i;
- err = iwch_register_mem(rhp, php, mhp, shift, pages);
- kfree(pages);
+
+ err = iwch_register_mem(rhp, php, mhp, shift);
if (err)
- goto err;
+ goto err_pbl;
- if (udata && t3b_device(rhp)) {
+ if (udata && !t3a_device(rhp)) {
uresp.pbl_addr = (mhp->attr.pbl_addr -
- rhp->rdev.rnic_info.pbl_base) >> 3;
+ rhp->rdev.rnic_info.pbl_base) >> 3;
PDBG("%s user resp pbl_addr 0x%x\n", __FUNCTION__,
uresp.pbl_addr);
@@ -661,6 +696,9 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
return &mhp->ibmr;
+err_pbl:
+ iwch_free_pbl(mhp);
+
err:
ib_umem_release(mhp->umem);
kfree(mhp);
@@ -819,8 +857,11 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
kfree(qhp);
return ERR_PTR(-ENOMEM);
}
+
attrs->cap.max_recv_wr = rqsize - 1;
attrs->cap.max_send_wr = sqsize;
+ attrs->cap.max_inline_data = T3_MAX_INLINE;
+
qhp->rhp = rhp;
qhp->attr.pd = php->pdid;
qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
@@ -995,7 +1036,7 @@ static int iwch_query_device(struct ib_device *ibdev,
props->device_cap_flags = dev->device_cap_flags;
props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
- props->max_mr_size = ~0ull;
+ props->max_mr_size = dev->attr.max_mr_size;
props->max_qp = dev->attr.max_qps;
props->max_qp_wr = dev->attr.max_wrs;
props->max_sge = dev->attr.max_sge_per_wr;
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.h b/drivers/infiniband/hw/cxgb3/iwch_provider.h
index 48833f3..8d42a3b 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.h
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.h
@@ -118,6 +118,7 @@ enum IWCH_QP_FLAGS {
};
struct iwch_mpa_attributes {
+ u8 initiator;
u8 recv_marker_enabled;
u8 xmit_marker_enabled; /* iWARP: enable inbound Read Resp. */
u8 crc_enabled;
@@ -322,6 +323,7 @@ enum iwch_qp_query_flags {
IWCH_QP_QUERY_TEST_USERWRITE = 0x32 /* Test special */
};
+u16 iwch_rqes_posted(struct iwch_qp *qhp);
int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
struct ib_send_wr **bad_wr);
int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
@@ -331,20 +333,21 @@ int iwch_bind_mw(struct ib_qp *qp,
struct ib_mw_bind *mw_bind);
int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg);
+int iwch_post_zb_read(struct iwch_qp *qhp);
int iwch_register_device(struct iwch_dev *dev);
void iwch_unregister_device(struct iwch_dev *dev);
int iwch_quiesce_qps(struct iwch_cq *chp);
int iwch_resume_qps(struct iwch_cq *chp);
void stop_read_rep_timer(struct iwch_qp *qhp);
int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
- struct iwch_mr *mhp,
- int shift,
- __be64 *page_list);
+ struct iwch_mr *mhp, int shift);
int iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php,
struct iwch_mr *mhp,
int shift,
- __be64 *page_list,
int npages);
+int iwch_alloc_pbl(struct iwch_mr *mhp, int npages);
+void iwch_free_pbl(struct iwch_mr *mhp);
+int iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset);
int build_phys_page_list(struct ib_phys_buf *buffer_list,
int num_phys_buf,
u64 *iova_start,
diff --git a/drivers/infiniband/hw/cxgb3/iwch_qp.c b/drivers/infiniband/hw/cxgb3/iwch_qp.c
index 9bb8112..353fbb3 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_qp.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_qp.c
@@ -586,6 +586,36 @@ static inline void build_term_codes(struct respQ_msg_t *rsp_msg,
}
}
+int iwch_post_zb_read(struct iwch_qp *qhp)
+{
+ union t3_wr *wqe;
+ struct sk_buff *skb;
+ u8 flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3;
+
+ PDBG("%s enter\n", __FUNCTION__);
+ skb = alloc_skb(40, GFP_KERNEL);
+ if (!skb) {
+ printk(KERN_ERR "%s cannot send zb_read!!\n", __FUNCTION__);
+ return -ENOMEM;
+ }
+ wqe = (union t3_wr *)skb_put(skb, sizeof(struct t3_rdma_read_wr));
+ memset(wqe, 0, sizeof(struct t3_rdma_read_wr));
+ wqe->read.rdmaop = T3_READ_REQ;
+ wqe->read.reserved[0] = 0;
+ wqe->read.reserved[1] = 0;
+ wqe->read.reserved[2] = 0;
+ wqe->read.rem_stag = cpu_to_be32(1);
+ wqe->read.rem_to = cpu_to_be64(1);
+ wqe->read.local_stag = cpu_to_be32(1);
+ wqe->read.local_len = cpu_to_be32(0);
+ wqe->read.local_to = cpu_to_be64(1);
+ wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_READ));
+ wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)|
+ V_FW_RIWR_LEN(flit_cnt));
+ skb->priority = CPL_PRIORITY_DATA;
+ return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb);
+}
+
/*
* This posts a TERMINATE with layer=RDMA, type=catastrophic.
*/
@@ -625,6 +655,7 @@ static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag)
{
struct iwch_cq *rchp, *schp;
int count;
+ int flushed;
rchp = get_chp(qhp->rhp, qhp->attr.rcq);
schp = get_chp(qhp->rhp, qhp->attr.scq);
@@ -639,18 +670,22 @@ static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag)
spin_lock(&qhp->lock);
cxio_flush_hw_cq(&rchp->cq);
cxio_count_rcqes(&rchp->cq, &qhp->wq, &count);
- cxio_flush_rq(&qhp->wq, &rchp->cq, count);
+ flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count);
spin_unlock(&qhp->lock);
spin_unlock_irqrestore(&rchp->lock, *flag);
+ if (flushed)
+ (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context);
/* locking heirarchy: cq lock first, then qp lock. */
spin_lock_irqsave(&schp->lock, *flag);
spin_lock(&qhp->lock);
cxio_flush_hw_cq(&schp->cq);
cxio_count_scqes(&schp->cq, &qhp->wq, &count);
- cxio_flush_sq(&qhp->wq, &schp->cq, count);
+ flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count);
spin_unlock(&qhp->lock);
spin_unlock_irqrestore(&schp->lock, *flag);
+ if (flushed)
+ (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context);
/* deref */
if (atomic_dec_and_test(&qhp->refcnt))
@@ -661,7 +696,7 @@ static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag)
static void flush_qp(struct iwch_qp *qhp, unsigned long *flag)
{
- if (t3b_device(qhp->rhp))
+ if (qhp->ibqp.uobject)
cxio_set_wq_in_error(&qhp->wq);
else
__flush_qp(qhp, flag);
@@ -669,11 +704,18 @@ static void flush_qp(struct iwch_qp *qhp, unsigned long *flag)
/*
- * Return non zero if at least one RECV was pre-posted.
+ * Return count of RECV WRs posted
*/
-static int rqes_posted(struct iwch_qp *qhp)
+u16 iwch_rqes_posted(struct iwch_qp *qhp)
{
- return fw_riwrh_opcode((struct fw_riwrh *)qhp->wq.queue) == T3_WR_RCV;
+ union t3_wr *wqe = qhp->wq.queue;
+ u16 count = 0;
+ while ((count+1) != 0 && fw_riwrh_opcode((struct fw_riwrh *)wqe) == T3_WR_RCV) {
+ count++;
+ wqe++;
+ }
+ PDBG("%s qhp %p count %u\n", __FUNCTION__, qhp, count);
+ return count;
}
static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp,
@@ -714,7 +756,17 @@ static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp,
init_attr.ird = qhp->attr.max_ird;
init_attr.qp_dma_addr = qhp->wq.dma_addr;
init_attr.qp_dma_size = (1UL << qhp->wq.size_log2);
- init_attr.flags = rqes_posted(qhp) ? RECVS_POSTED : 0;
+ init_attr.rqe_count = iwch_rqes_posted(qhp);
+ init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0;
+ init_attr.flags |= capable(CAP_NET_BIND_SERVICE) ? PRIV_QP : 0;
+ if (peer2peer) {
+ init_attr.rtr_type = RTR_READ;
+ if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator)
+ init_attr.ord = 1;
+ if (init_attr.ird == 0 && !qhp->attr.mpa_attr.initiator)
+ init_attr.ird = 1;
+ } else
+ init_attr.rtr_type = 0;
init_attr.irs = qhp->ep->rcv_seq;
PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d "
"flags 0x%x qpcaps 0x%x\n", __FUNCTION__,
@@ -829,11 +881,12 @@ int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp,
abort=0;
disconnect = 1;
ep = qhp->ep;
+ get_ep(&ep->com);
}
break;
case IWCH_QP_STATE_TERMINATE:
qhp->attr.state = IWCH_QP_STATE_TERMINATE;
- if (t3b_device(qhp->rhp))
+ if (qhp->ibqp.uobject)
cxio_set_wq_in_error(&qhp->wq);
if (!internal)
terminate = 1;
@@ -844,6 +897,7 @@ int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp,
abort=1;
disconnect = 1;
ep = qhp->ep;
+ get_ep(&ep->com);
}
goto err;
break;
@@ -859,6 +913,7 @@ int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp,
}
switch (attrs->next_state) {
case IWCH_QP_STATE_IDLE:
+ flush_qp(qhp, &flag);
qhp->attr.state = IWCH_QP_STATE_IDLE;
qhp->attr.llp_stream_handle = NULL;
put_ep(&qhp->ep->com);
@@ -925,8 +980,10 @@ out:
* on the EP. This can be a normal close (RTS->CLOSING) or
* an abnormal close (RTS/CLOSING->ERROR).
*/
- if (disconnect)
+ if (disconnect) {
iwch_ep_disconnect(ep, abort, GFP_KERNEL);
+ put_ep(&ep->com);
+ }
/*
* If free is 1, then we've disassociated the EP from the QP
diff --git a/drivers/infiniband/hw/ehca/ehca_av.c b/drivers/infiniband/hw/ehca/ehca_av.c
index d53bca7..dd4de2f 100644
--- a/drivers/infiniband/hw/ehca/ehca_av.c
+++ b/drivers/infiniband/hw/ehca/ehca_av.c
@@ -41,9 +41,6 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-
-#include <asm/current.h>
-
#include "ehca_tools.h"
#include "ehca_iverbs.h"
#include "hcp_if.h"
@@ -170,17 +167,8 @@ int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
{
struct ehca_av *av;
struct ehca_ud_av new_ehca_av;
- struct ehca_pd *my_pd = container_of(ah->pd, struct ehca_pd, ib_pd);
struct ehca_shca *shca = container_of(ah->pd->device, struct ehca_shca,
ib_device);
- u32 cur_pid = current->tgid;
-
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(ah->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
memset(&new_ehca_av, 0, sizeof(new_ehca_av));
new_ehca_av.sl = ah_attr->sl;
@@ -242,15 +230,6 @@ int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
{
struct ehca_av *av = container_of(ah, struct ehca_av, ib_ah);
- struct ehca_pd *my_pd = container_of(ah->pd, struct ehca_pd, ib_pd);
- u32 cur_pid = current->tgid;
-
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(ah->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
memcpy(&ah_attr->grh.dgid, &av->av.grh.word_3,
sizeof(ah_attr->grh.dgid));
@@ -273,16 +252,6 @@ int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
int ehca_destroy_ah(struct ib_ah *ah)
{
- struct ehca_pd *my_pd = container_of(ah->pd, struct ehca_pd, ib_pd);
- u32 cur_pid = current->tgid;
-
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(ah->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
-
kmem_cache_free(av_cache, container_of(ah, struct ehca_av, ib_ah));
return 0;
diff --git a/drivers/infiniband/hw/ehca/ehca_classes.h b/drivers/infiniband/hw/ehca/ehca_classes.h
index 1b259cd..0de5982 100644
--- a/drivers/infiniband/hw/ehca/ehca_classes.h
+++ b/drivers/infiniband/hw/ehca/ehca_classes.h
@@ -135,7 +135,6 @@ struct ehca_shca {
struct ehca_pd {
struct ib_pd ib_pd;
struct ipz_pd fw_pd;
- u32 ownpid;
/* small queue mgmt */
struct mutex lock;
struct list_head free[2];
@@ -172,6 +171,7 @@ struct ehca_qp {
};
u32 qp_type;
enum ehca_ext_qp_type ext_type;
+ enum ib_qp_state state;
struct ipz_queue ipz_squeue;
struct ehca_qmap_entry *sq_map;
struct ipz_queue ipz_rqueue;
@@ -203,6 +203,7 @@ struct ehca_qp {
u32 packet_count;
atomic_t nr_events; /* events seen */
wait_queue_head_t wait_completion;
+ int mig_armed;
};
#define IS_SRQ(qp) (qp->ext_type == EQPT_SRQ)
@@ -229,7 +230,6 @@ struct ehca_cq {
atomic_t nr_events; /* #events seen */
wait_queue_head_t wait_completion;
spinlock_t task_lock;
- u32 ownpid;
/* mmap counter for resources mapped into user space */
u32 mm_count_queue;
u32 mm_count_galpa;
diff --git a/drivers/infiniband/hw/ehca/ehca_cq.c b/drivers/infiniband/hw/ehca/ehca_cq.c
index 776e2d0..5540b27 100644
--- a/drivers/infiniband/hw/ehca/ehca_cq.c
+++ b/drivers/infiniband/hw/ehca/ehca_cq.c
@@ -43,8 +43,6 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#include <asm/current.h>
-
#include "ehca_iverbs.h"
#include "ehca_classes.h"
#include "ehca_irq.h"
@@ -157,7 +155,6 @@ struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector,
spin_lock_init(&my_cq->task_lock);
atomic_set(&my_cq->nr_events, 0);
init_waitqueue_head(&my_cq->wait_completion);
- my_cq->ownpid = current->tgid;
cq = &my_cq->ib_cq;
@@ -330,7 +327,6 @@ int ehca_destroy_cq(struct ib_cq *cq)
struct ehca_shca *shca = container_of(device, struct ehca_shca,
ib_device);
struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
- u32 cur_pid = current->tgid;
unsigned long flags;
if (cq->uobject) {
@@ -339,12 +335,6 @@ int ehca_destroy_cq(struct ib_cq *cq)
"user space cq_num=%x", my_cq->cq_number);
return -EINVAL;
}
- if (my_cq->ownpid != cur_pid) {
- ehca_err(device, "Invalid caller pid=%x ownpid=%x "
- "cq_num=%x",
- cur_pid, my_cq->ownpid, my_cq->cq_number);
- return -EINVAL;
- }
}
/*
@@ -385,15 +375,6 @@ int ehca_destroy_cq(struct ib_cq *cq)
int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
{
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- u32 cur_pid = current->tgid;
-
- if (cq->uobject && my_cq->ownpid != cur_pid) {
- ehca_err(cq->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_cq->ownpid);
- return -EINVAL;
- }
-
/* TODO: proper resize needs to be done */
ehca_err(cq->device, "not implemented yet");
diff --git a/drivers/infiniband/hw/ehca/ehca_eq.c b/drivers/infiniband/hw/ehca/ehca_eq.c
index 1d41faa..aab80e9 100644
--- a/drivers/infiniband/hw/ehca/ehca_eq.c
+++ b/drivers/infiniband/hw/ehca/ehca_eq.c
@@ -54,7 +54,8 @@ int ehca_create_eq(struct ehca_shca *shca,
struct ehca_eq *eq,
const enum ehca_eq_type type, const u32 length)
{
- u64 ret;
+ int ret;
+ u64 h_ret;
u32 nr_pages;
u32 i;
void *vpage;
@@ -73,15 +74,15 @@ int ehca_create_eq(struct ehca_shca *shca,
return -EINVAL;
}
- ret = hipz_h_alloc_resource_eq(shca->ipz_hca_handle,
- &eq->pf,
- type,
- length,
- &eq->ipz_eq_handle,
- &eq->length,
- &nr_pages, &eq->ist);
+ h_ret = hipz_h_alloc_resource_eq(shca->ipz_hca_handle,
+ &eq->pf,
+ type,
+ length,
+ &eq->ipz_eq_handle,
+ &eq->length,
+ &nr_pages, &eq->ist);
- if (ret != H_SUCCESS) {
+ if (h_ret != H_SUCCESS) {
ehca_err(ib_dev, "Can't allocate EQ/NEQ. eq=%p", eq);
return -EINVAL;
}
@@ -97,24 +98,22 @@ int ehca_create_eq(struct ehca_shca *shca,
u64 rpage;
vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
- if (!vpage) {
- ret = H_RESOURCE;
+ if (!vpage)
goto create_eq_exit2;
- }
rpage = virt_to_abs(vpage);
- ret = hipz_h_register_rpage_eq(shca->ipz_hca_handle,
- eq->ipz_eq_handle,
- &eq->pf,
- 0, 0, rpage, 1);
+ h_ret = hipz_h_register_rpage_eq(shca->ipz_hca_handle,
+ eq->ipz_eq_handle,
+ &eq->pf,
+ 0, 0, rpage, 1);
if (i == (nr_pages - 1)) {
/* last page */
vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
- if (ret != H_SUCCESS || vpage)
+ if (h_ret != H_SUCCESS || vpage)
goto create_eq_exit2;
} else {
- if (ret != H_PAGE_REGISTERED || !vpage)
+ if (h_ret != H_PAGE_REGISTERED || !vpage)
goto create_eq_exit2;
}
}
diff --git a/drivers/infiniband/hw/ehca/ehca_hca.c b/drivers/infiniband/hw/ehca/ehca_hca.c
index c04cbb1..6e5ccd0 100644
--- a/drivers/infiniband/hw/ehca/ehca_hca.c
+++ b/drivers/infiniband/hw/ehca/ehca_hca.c
@@ -43,6 +43,11 @@
#include "ehca_iverbs.h"
#include "hcp_if.h"
+static unsigned int limit_uint(unsigned int value)
+{
+ return min_t(unsigned int, value, INT_MAX);
+}
+
int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props)
{
int i, ret = 0;
@@ -83,38 +88,41 @@ int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props)
props->vendor_id = rblock->vendor_id >> 8;
props->vendor_part_id = rblock->vendor_part_id >> 16;
props->hw_ver = rblock->hw_ver;
- props->max_qp = min_t(unsigned, rblock->max_qp, INT_MAX);
- props->max_qp_wr = min_t(unsigned, rblock->max_wqes_wq, INT_MAX);
- props->max_sge = min_t(unsigned, rblock->max_sge, INT_MAX);
- props->max_sge_rd = min_t(unsigned, rblock->max_sge_rd, INT_MAX);
- props->max_cq = min_t(unsigned, rblock->max_cq, INT_MAX);
- props->max_cqe = min_t(unsigned, rblock->max_cqe, INT_MAX);
- props->max_mr = min_t(unsigned, rblock->max_mr, INT_MAX);
- props->max_mw = min_t(unsigned, rblock->max_mw, INT_MAX);
- props->max_pd = min_t(unsigned, rblock->max_pd, INT_MAX);
- props->max_ah = min_t(unsigned, rblock->max_ah, INT_MAX);
- props->max_fmr = min_t(unsigned, rblock->max_mr, INT_MAX);
+ props->max_qp = limit_uint(rblock->max_qp);
+ props->max_qp_wr = limit_uint(rblock->max_wqes_wq);
+ props->max_sge = limit_uint(rblock->max_sge);
+ props->max_sge_rd = limit_uint(rblock->max_sge_rd);
+ props->max_cq = limit_uint(rblock->max_cq);
+ props->max_cqe = limit_uint(rblock->max_cqe);
+ props->max_mr = limit_uint(rblock->max_mr);
+ props->max_mw = limit_uint(rblock->max_mw);
+ props->max_pd = limit_uint(rblock->max_pd);
+ props->max_ah = limit_uint(rblock->max_ah);
+ props->max_ee = limit_uint(rblock->max_rd_ee_context);
+ props->max_rdd = limit_uint(rblock->max_rd_domain);
+ props->max_fmr = limit_uint(rblock->max_mr);
+ props->max_qp_rd_atom = limit_uint(rblock->max_rr_qp);
+ props->max_ee_rd_atom = limit_uint(rblock->max_rr_ee_context);
+ props->max_res_rd_atom = limit_uint(rblock->max_rr_hca);
+ props->max_qp_init_rd_atom = limit_uint(rblock->max_act_wqs_qp);
+ props->max_ee_init_rd_atom = limit_uint(rblock->max_act_wqs_ee_context);
if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
- props->max_srq = props->max_qp;
- props->max_srq_wr = props->max_qp_wr;
+ props->max_srq = limit_uint(props->max_qp);
+ props->max_srq_wr = limit_uint(props->max_qp_wr);
props->max_srq_sge = 3;
}
- props->max_pkeys = 16;
+ props->max_pkeys = 16;
/* Some FW versions say 0 here; insert sensible value in that case */
props->local_ca_ack_delay = rblock->local_ca_ack_delay ?
min_t(u8, rblock->local_ca_ack_delay, 255) : 12;
- props->max_raw_ipv6_qp
- = min_t(unsigned, rblock->max_raw_ipv6_qp, INT_MAX);
- props->max_raw_ethy_qp
- = min_t(unsigned, rblock->max_raw_ethy_qp, INT_MAX);
- props->max_mcast_grp
- = min_t(unsigned, rblock->max_mcast_grp, INT_MAX);
- props->max_mcast_qp_attach
- = min_t(unsigned, rblock->max_mcast_qp_attach, INT_MAX);
+ props->max_raw_ipv6_qp = limit_uint(rblock->max_raw_ipv6_qp);
+ props->max_raw_ethy_qp = limit_uint(rblock->max_raw_ethy_qp);
+ props->max_mcast_grp = limit_uint(rblock->max_mcast_grp);
+ props->max_mcast_qp_attach = limit_uint(rblock->max_mcast_qp_attach);
props->max_total_mcast_qp_attach
- = min_t(unsigned, rblock->max_total_mcast_qp_attach, INT_MAX);
+ = limit_uint(rblock->max_total_mcast_qp_attach);
/* translate device capabilities */
props->device_cap_flags = IB_DEVICE_SYS_IMAGE_GUID |
@@ -129,6 +137,46 @@ query_device1:
return ret;
}
+static enum ib_mtu map_mtu(struct ehca_shca *shca, u32 fw_mtu)
+{
+ switch (fw_mtu) {
+ case 0x1:
+ return IB_MTU_256;
+ case 0x2:
+ return IB_MTU_512;
+ case 0x3:
+ return IB_MTU_1024;
+ case 0x4:
+ return IB_MTU_2048;
+ case 0x5:
+ return IB_MTU_4096;
+ default:
+ ehca_err(&shca->ib_device, "Unknown MTU size: %x.",
+ fw_mtu);
+ return 0;
+ }
+}
+
+static u8 map_number_of_vls(struct ehca_shca *shca, u32 vl_cap)
+{
+ switch (vl_cap) {
+ case 0x1:
+ return 1;
+ case 0x2:
+ return 2;
+ case 0x3:
+ return 4;
+ case 0x4:
+ return 8;
+ case 0x5:
+ return 15;
+ default:
+ ehca_err(&shca->ib_device, "invalid Vl Capability: %x.",
+ vl_cap);
+ return 0;
+ }
+}
+
int ehca_query_port(struct ib_device *ibdev,
u8 port, struct ib_port_attr *props)
{
@@ -153,31 +201,13 @@ int ehca_query_port(struct ib_device *ibdev,
memset(props, 0, sizeof(struct ib_port_attr));
- switch (rblock->max_mtu) {
- case 0x1:
- props->active_mtu = props->max_mtu = IB_MTU_256;
- break;
- case 0x2:
- props->active_mtu = props->max_mtu = IB_MTU_512;
- break;
- case 0x3:
- props->active_mtu = props->max_mtu = IB_MTU_1024;
- break;
- case 0x4:
- props->active_mtu = props->max_mtu = IB_MTU_2048;
- break;
- case 0x5:
- props->active_mtu = props->max_mtu = IB_MTU_4096;
- break;
- default:
- ehca_err(&shca->ib_device, "Unknown MTU size: %x.",
- rblock->max_mtu);
- break;
- }
-
+ props->active_mtu = props->max_mtu = map_mtu(shca, rblock->max_mtu);
props->port_cap_flags = rblock->capability_mask;
props->gid_tbl_len = rblock->gid_tbl_len;
- props->max_msg_sz = rblock->max_msg_sz;
+ if (rblock->max_msg_sz)
+ props->max_msg_sz = rblock->max_msg_sz;
+ else
+ props->max_msg_sz = 0x1 << 31;
props->bad_pkey_cntr = rblock->bad_pkey_cntr;
props->qkey_viol_cntr = rblock->qkey_viol_cntr;
props->pkey_tbl_len = rblock->pkey_tbl_len;
@@ -187,6 +217,7 @@ int ehca_query_port(struct ib_device *ibdev,
props->sm_sl = rblock->sm_sl;
props->subnet_timeout = rblock->subnet_timeout;
props->init_type_reply = rblock->init_type_reply;
+ props->max_vl_num = map_number_of_vls(shca, rblock->vl_cap);
if (rblock->state && rblock->phys_width) {
props->phys_state = rblock->phys_pstate;
@@ -315,7 +346,7 @@ query_gid1:
return ret;
}
-const u32 allowed_port_caps = (
+static const u32 allowed_port_caps = (
IB_PORT_SM | IB_PORT_LED_INFO_SUP | IB_PORT_CM_SUP |
IB_PORT_SNMP_TUNNEL_SUP | IB_PORT_DEVICE_MGMT_SUP |
IB_PORT_VENDOR_CLASS_SUP);
diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c
index 8c821d4..99642a6 100644
--- a/drivers/infiniband/hw/ehca/ehca_irq.c
+++ b/drivers/infiniband/hw/ehca/ehca_irq.c
@@ -178,6 +178,10 @@ static void dispatch_qp_event(struct ehca_shca *shca, struct ehca_qp *qp,
{
struct ib_event event;
+ /* PATH_MIG without the QP ever having been armed is false alarm */
+ if (event_type == IB_EVENT_PATH_MIG && !qp->mig_armed)
+ return;
+
event.device = &shca->ib_device;
event.event = event_type;
@@ -642,7 +646,7 @@ static inline int find_next_online_cpu(struct ehca_comp_pool *pool)
unsigned long flags;
WARN_ON_ONCE(!in_interrupt());
- if (ehca_debug_level)
+ if (ehca_debug_level >= 3)
ehca_dmp(&cpu_online_map, sizeof(cpumask_t), "");
spin_lock_irqsave(&pool->last_cpu_lock, flags);
diff --git a/drivers/infiniband/hw/ehca/ehca_main.c b/drivers/infiniband/hw/ehca/ehca_main.c
index b9f2b9f..b359aed 100644
--- a/drivers/infiniband/hw/ehca/ehca_main.c
+++ b/drivers/infiniband/hw/ehca/ehca_main.c
@@ -50,68 +50,63 @@
#include "ehca_tools.h"
#include "hcp_if.h"
-#define HCAD_VERSION "SVNEHCA_0025"
+#define HCAD_VERSION "SVNEHCA_0026"
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
MODULE_DESCRIPTION("IBM eServer HCA InfiniBand Device Driver");
MODULE_VERSION(HCAD_VERSION);
-int ehca_open_aqp1 = 0;
+static int ehca_open_aqp1 = 0;
+static int ehca_hw_level = 0;
+static int ehca_poll_all_eqs = 1;
+
int ehca_debug_level = 0;
-int ehca_hw_level = 0;
int ehca_nr_ports = 2;
int ehca_use_hp_mr = 0;
int ehca_port_act_time = 30;
-int ehca_poll_all_eqs = 1;
int ehca_static_rate = -1;
int ehca_scaling_code = 0;
-int ehca_mr_largepage = 1;
int ehca_lock_hcalls = -1;
int ehca_max_cq = -1;
int ehca_max_qp = -1;
-module_param_named(open_aqp1, ehca_open_aqp1, int, S_IRUGO);
-module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
-module_param_named(hw_level, ehca_hw_level, int, S_IRUGO);
-module_param_named(nr_ports, ehca_nr_ports, int, S_IRUGO);
-module_param_named(use_hp_mr, ehca_use_hp_mr, int, S_IRUGO);
-module_param_named(port_act_time, ehca_port_act_time, int, S_IRUGO);
-module_param_named(poll_all_eqs, ehca_poll_all_eqs, int, S_IRUGO);
-module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
-module_param_named(scaling_code, ehca_scaling_code, int, S_IRUGO);
-module_param_named(mr_largepage, ehca_mr_largepage, int, S_IRUGO);
+module_param_named(open_aqp1, ehca_open_aqp1, bool, S_IRUGO);
+module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
+module_param_named(hw_level, ehca_hw_level, int, S_IRUGO);
+module_param_named(nr_ports, ehca_nr_ports, int, S_IRUGO);
+module_param_named(use_hp_mr, ehca_use_hp_mr, bool, S_IRUGO);
+module_param_named(port_act_time, ehca_port_act_time, int, S_IRUGO);
+module_param_named(poll_all_eqs, ehca_poll_all_eqs, bool, S_IRUGO);
+module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
+module_param_named(scaling_code, ehca_scaling_code, bool, S_IRUGO);
module_param_named(lock_hcalls, ehca_lock_hcalls, bool, S_IRUGO);
module_param_named(number_of_cqs, ehca_max_cq, int, S_IRUGO);
module_param_named(number_of_qps, ehca_max_qp, int, S_IRUGO);
MODULE_PARM_DESC(open_aqp1,
- "AQP1 on startup (0: no (default), 1: yes)");
+ "Open AQP1 on startup (default: no)");
MODULE_PARM_DESC(debug_level,
- "debug level"
- " (0: no debug traces (default), 1: with debug traces)");
+ "Amount of debug output (0: none (default), 1: traces, "
+ "2: some dumps, 3: lots)");
MODULE_PARM_DESC(hw_level,
- "hardware level"
- " (0: autosensing (default), 1: v. 0.20, 2: v. 0.21)");
+ "Hardware level (0: autosensing (default), "
+ "0x10..0x14: eHCA, 0x20..0x23: eHCA2)");
MODULE_PARM_DESC(nr_ports,
"number of connected ports (-1: autodetect, 1: port one only, "
"2: two ports (default)");
MODULE_PARM_DESC(use_hp_mr,
- "high performance MRs (0: no (default), 1: yes)");
+ "Use high performance MRs (default: no)");
MODULE_PARM_DESC(port_act_time,
- "time to wait for port activation (default: 30 sec)");
+ "Time to wait for port activation (default: 30 sec)");
MODULE_PARM_DESC(poll_all_eqs,
- "polls all event queues periodically"
- " (0: no, 1: yes (default))");
+ "Poll all event queues periodically (default: yes)");
MODULE_PARM_DESC(static_rate,
- "set permanent static rate (default: disabled)");
+ "Set permanent static rate (default: no static rate)");
MODULE_PARM_DESC(scaling_code,
- "set scaling code (0: disabled/default, 1: enabled)");
-MODULE_PARM_DESC(mr_largepage,
- "use large page for MR (0: use PAGE_SIZE (default), "
- "1: use large page depending on MR size");
+ "Enable scaling code (default: no)");
MODULE_PARM_DESC(lock_hcalls,
- "serialize all hCalls made by the driver "
+ "Serialize all hCalls made by the driver "
"(default: autodetect)");
MODULE_PARM_DESC(number_of_cqs,
"Max number of CQs which can be allocated "
@@ -284,6 +279,7 @@ static int ehca_sense_attributes(struct ehca_shca *shca)
u64 h_ret;
struct hipz_query_hca *rblock;
struct hipz_query_port *port;
+ const char *loc_code;
static const u32 pgsize_map[] = {
HCA_CAP_MR_PGSIZE_4K, 0x1000,
@@ -292,6 +288,12 @@ static int ehca_sense_attributes(struct ehca_shca *shca)
HCA_CAP_MR_PGSIZE_16M, 0x1000000,
};
+ ehca_gen_dbg("Probing adapter %s...",
+ shca->ibmebus_dev->ofdev.node->full_name);
+ loc_code = of_get_property(shca->ibmebus_dev->ofdev.node, "ibm,loc-code", NULL);
+ if (loc_code)
+ ehca_gen_dbg(" ... location lode=%s", loc_code);
+
rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
if (!rblock) {
ehca_gen_err("Cannot allocate rblock memory.");
@@ -359,11 +361,9 @@ static int ehca_sense_attributes(struct ehca_shca *shca)
/* translate supported MR page sizes; always support 4K */
shca->hca_cap_mr_pgsize = EHCA_PAGESIZE;
- if (ehca_mr_largepage) { /* support extra sizes only if enabled */
- for (i = 0; i < ARRAY_SIZE(pgsize_map); i += 2)
- if (rblock->memory_page_size_supported & pgsize_map[i])
- shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];
- }
+ for (i = 0; i < ARRAY_SIZE(pgsize_map); i += 2)
+ if (rblock->memory_page_size_supported & pgsize_map[i])
+ shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];
/* Set maximum number of CQs and QPs to calculate EQ size */
if (ehca_max_qp == -1)
@@ -425,7 +425,7 @@ init_node_guid1:
return ret;
}
-int ehca_init_device(struct ehca_shca *shca)
+static int ehca_init_device(struct ehca_shca *shca)
{
int ret;
@@ -595,8 +595,7 @@ static int ehca_destroy_aqp1(struct ehca_sport *sport)
static ssize_t ehca_show_debug_level(struct device_driver *ddp, char *buf)
{
- return snprintf(buf, PAGE_SIZE, "%d\n",
- ehca_debug_level);
+ return snprintf(buf, PAGE_SIZE, "%d\n", ehca_debug_level);
}
static ssize_t ehca_store_debug_level(struct device_driver *ddp,
@@ -608,8 +607,8 @@ static ssize_t ehca_store_debug_level(struct device_driver *ddp,
return 1;
}
-DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
- ehca_show_debug_level, ehca_store_debug_level);
+static DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
+ ehca_show_debug_level, ehca_store_debug_level);
static struct attribute *ehca_drv_attrs[] = {
&driver_attr_debug_level.attr,
@@ -680,14 +679,6 @@ static ssize_t ehca_show_adapter_handle(struct device *dev,
}
static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
-static ssize_t ehca_show_mr_largepage(struct device *dev,
- struct device_attribute *attr,
- char *buf)
-{
- return sprintf(buf, "%d\n", ehca_mr_largepage);
-}
-static DEVICE_ATTR(mr_largepage, S_IRUGO, ehca_show_mr_largepage, NULL);
-
static struct attribute *ehca_dev_attrs[] = {
&dev_attr_adapter_handle.attr,
&dev_attr_num_ports.attr,
@@ -704,7 +695,6 @@ static struct attribute *ehca_dev_attrs[] = {
&dev_attr_cur_mw.attr,
&dev_attr_max_pd.attr,
&dev_attr_max_ah.attr,
- &dev_attr_mr_largepage.attr,
NULL
};
@@ -965,7 +955,7 @@ void ehca_poll_eqs(unsigned long data)
spin_unlock(&shca_list_lock);
}
-int __init ehca_module_init(void)
+static int __init ehca_module_init(void)
{
int ret;
@@ -1016,7 +1006,7 @@ module_init1:
return ret;
};
-void __exit ehca_module_exit(void)
+static void __exit ehca_module_exit(void)
{
if (ehca_poll_all_eqs == 1)
del_timer_sync(&poll_eqs_timer);
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.c b/drivers/infiniband/hw/ehca/ehca_mrmw.c
index e239bbf..46ae4eb 100644
--- a/drivers/infiniband/hw/ehca/ehca_mrmw.c
+++ b/drivers/infiniband/hw/ehca/ehca_mrmw.c
@@ -40,8 +40,6 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#include <asm/current.h>
-
#include <rdma/ib_umem.h>
#include "ehca_iverbs.h"
@@ -419,7 +417,6 @@ int ehca_rereg_phys_mr(struct ib_mr *mr,
struct ehca_shca *shca =
container_of(mr->device, struct ehca_shca, ib_device);
struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
- struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
u64 new_size;
u64 *new_start;
u32 new_acl;
@@ -429,15 +426,6 @@ int ehca_rereg_phys_mr(struct ib_mr *mr,
u32 num_kpages = 0;
u32 num_hwpages = 0;
struct ehca_mr_pginfo pginfo;
- u32 cur_pid = current->tgid;
-
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- (my_pd->ownpid != cur_pid)) {
- ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
/* TODO not supported, because PHYP rereg hCall needs pages */
@@ -577,19 +565,9 @@ int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
struct ehca_shca *shca =
container_of(mr->device, struct ehca_shca, ib_device);
struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
- struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
- u32 cur_pid = current->tgid;
unsigned long sl_flags;
struct ehca_mr_hipzout_parms hipzout;
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- (my_pd->ownpid != cur_pid)) {
- ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- ret = -EINVAL;
- goto query_mr_exit0;
- }
-
if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
"e_mr->flags=%x", mr, e_mr, e_mr->flags);
@@ -634,16 +612,6 @@ int ehca_dereg_mr(struct ib_mr *mr)
struct ehca_shca *shca =
container_of(mr->device, struct ehca_shca, ib_device);
struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
- struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
- u32 cur_pid = current->tgid;
-
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- (my_pd->ownpid != cur_pid)) {
- ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- ret = -EINVAL;
- goto dereg_mr_exit0;
- }
if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
@@ -1826,8 +1794,9 @@ static int ehca_check_kpages_per_ate(struct scatterlist *page_list,
int t;
for (t = start_idx; t <= end_idx; t++) {
u64 pgaddr = page_to_pfn(sg_page(&page_list[t])) << PAGE_SHIFT;
- ehca_gen_dbg("chunk_page=%lx value=%016lx", pgaddr,
- *(u64 *)abs_to_virt(phys_to_abs(pgaddr)));
+ if (ehca_debug_level >= 3)
+ ehca_gen_dbg("chunk_page=%lx value=%016lx", pgaddr,
+ *(u64 *)abs_to_virt(phys_to_abs(pgaddr)));
if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
ehca_gen_err("uncontiguous page found pgaddr=%lx "
"prev_pgaddr=%lx page_list_i=%x",
@@ -1894,10 +1863,13 @@ static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
pgaddr &
~(pginfo->hwpage_size - 1));
}
- ehca_gen_dbg("kpage=%lx chunk_page=%lx "
- "value=%016lx", *kpage, pgaddr,
- *(u64 *)abs_to_virt(
- phys_to_abs(pgaddr)));
+ if (ehca_debug_level >= 3) {
+ u64 val = *(u64 *)abs_to_virt(
+ phys_to_abs(pgaddr));
+ ehca_gen_dbg("kpage=%lx chunk_page=%lx "
+ "value=%016lx",
+ *kpage, pgaddr, val);
+ }
prev_pgaddr = pgaddr;
i++;
pginfo->kpage_cnt++;
@@ -1952,9 +1924,8 @@ next_kpage:
return ret;
}
-int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
+static int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
+ u32 number, u64 *kpage)
{
int ret = 0;
struct ib_phys_buf *pbuf;
@@ -2012,9 +1983,8 @@ int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
return ret;
}
-int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
+static int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
+ u32 number, u64 *kpage)
{
int ret = 0;
u64 *fmrlist;
diff --git a/drivers/infiniband/hw/ehca/ehca_pd.c b/drivers/infiniband/hw/ehca/ehca_pd.c
index 43bcf08..2fe5548 100644
--- a/drivers/infiniband/hw/ehca/ehca_pd.c
+++ b/drivers/infiniband/hw/ehca/ehca_pd.c
@@ -38,8 +38,6 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#include <asm/current.h>
-
#include "ehca_tools.h"
#include "ehca_iverbs.h"
@@ -58,7 +56,6 @@ struct ib_pd *ehca_alloc_pd(struct ib_device *device,
return ERR_PTR(-ENOMEM);
}
- pd->ownpid = current->tgid;
for (i = 0; i < 2; i++) {
INIT_LIST_HEAD(&pd->free[i]);
INIT_LIST_HEAD(&pd->full[i]);
@@ -85,18 +82,10 @@ struct ib_pd *ehca_alloc_pd(struct ib_device *device,
int ehca_dealloc_pd(struct ib_pd *pd)
{
- u32 cur_pid = current->tgid;
struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
int i, leftovers = 0;
struct ipz_small_queue_page *page, *tmp;
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(pd->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
-
for (i = 0; i < 2; i++) {
list_splice(&my_pd->full[i], &my_pd->free[i]);
list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) {
diff --git a/drivers/infiniband/hw/ehca/ehca_qp.c b/drivers/infiniband/hw/ehca/ehca_qp.c
index a552639..f60e425 100644
--- a/drivers/infiniband/hw/ehca/ehca_qp.c
+++ b/drivers/infiniband/hw/ehca/ehca_qp.c
@@ -43,9 +43,6 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-
-#include <asm/current.h>
-
#include "ehca_classes.h"
#include "ehca_tools.h"
#include "ehca_qes.h"
@@ -571,6 +568,7 @@ static struct ehca_qp *internal_create_qp(
spin_lock_init(&my_qp->spinlock_r);
my_qp->qp_type = qp_type;
my_qp->ext_type = parms.ext_type;
+ my_qp->state = IB_QPS_RESET;
if (init_attr->recv_cq)
my_qp->recv_cq =
@@ -1000,7 +998,7 @@ static int prepare_sqe_rts(struct ehca_qp *my_qp, struct ehca_shca *shca,
qp_num, bad_send_wqe_p);
/* convert wqe pointer to vadr */
bad_send_wqe_v = abs_to_virt((u64)bad_send_wqe_p);
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(bad_send_wqe_v, 32, "qp_num=%x bad_wqe", qp_num);
squeue = &my_qp->ipz_squeue;
if (ipz_queue_abs_to_offset(squeue, (u64)bad_send_wqe_p, &q_ofs)) {
@@ -1013,7 +1011,7 @@ static int prepare_sqe_rts(struct ehca_qp *my_qp, struct ehca_shca *shca,
wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
*bad_wqe_cnt = 0;
while (wqe->optype != 0xff && wqe->wqef != 0xff) {
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(wqe, 32, "qp_num=%x wqe", qp_num);
wqe->nr_of_data_seg = 0; /* suppress data access */
wqe->wqef = WQEF_PURGE; /* WQE to be purged */
@@ -1471,6 +1469,8 @@ static int internal_modify_qp(struct ib_qp *ibqp,
goto modify_qp_exit2;
}
mqpcb->path_migration_state = attr->path_mig_state + 1;
+ if (attr->path_mig_state == IB_MIG_REARM)
+ my_qp->mig_armed = 1;
update_mask |=
EHCA_BMASK_SET(MQPCB_MASK_PATH_MIGRATION_STATE, 1);
}
@@ -1485,7 +1485,7 @@ static int internal_modify_qp(struct ib_qp *ibqp,
/* no support for max_send/recv_sge yet */
}
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(mqpcb, 4*70, "qp_num=%x", ibqp->qp_num);
h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
@@ -1543,6 +1543,8 @@ static int internal_modify_qp(struct ib_qp *ibqp,
if (attr_mask & IB_QP_QKEY)
my_qp->qkey = attr->qkey;
+ my_qp->state = qp_new_state;
+
modify_qp_exit2:
if (squeue_locked) { /* this means: sqe -> rts */
spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
@@ -1561,16 +1563,6 @@ int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
ib_device);
struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
- struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
- ib_pd);
- u32 cur_pid = current->tgid;
-
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(ibqp->pd->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
/* The if-block below caches qp_attr to be modified for GSI and SMI
* qps during the initialization by ib_mad. When the respective port
@@ -1671,23 +1663,13 @@ int ehca_query_qp(struct ib_qp *qp,
int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
{
struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
- struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
- ib_pd);
struct ehca_shca *shca = container_of(qp->device, struct ehca_shca,
ib_device);
struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
struct hcp_modify_qp_control_block *qpcb;
- u32 cur_pid = current->tgid;
int cnt, ret = 0;
u64 h_ret;
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(qp->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
-
if (qp_attr_mask & QP_ATTR_QUERY_NOT_SUPPORTED) {
ehca_err(qp->device, "Invalid attribute mask "
"ehca_qp=%p qp_num=%x qp_attr_mask=%x ",
@@ -1818,7 +1800,7 @@ int ehca_query_qp(struct ib_qp *qp,
if (qp_init_attr)
*qp_init_attr = my_qp->init_attr;
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(qpcb, 4*70, "qp_num=%x", qp->qp_num);
query_qp_exit1:
@@ -1832,8 +1814,6 @@ int ehca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
{
struct ehca_qp *my_qp =
container_of(ibsrq, struct ehca_qp, ib_srq);
- struct ehca_pd *my_pd =
- container_of(ibsrq->pd, struct ehca_pd, ib_pd);
struct ehca_shca *shca =
container_of(ibsrq->pd->device, struct ehca_shca, ib_device);
struct hcp_modify_qp_control_block *mqpcb;
@@ -1841,14 +1821,6 @@ int ehca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
u64 h_ret;
int ret = 0;
- u32 cur_pid = current->tgid;
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(ibsrq->pd->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
-
mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
if (!mqpcb) {
ehca_err(ibsrq->device, "Could not get zeroed page for mqpcb "
@@ -1876,7 +1848,7 @@ int ehca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
goto modify_srq_exit0;
}
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(mqpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
h_ret = hipz_h_modify_qp(shca->ipz_hca_handle, my_qp->ipz_qp_handle,
@@ -1899,22 +1871,13 @@ modify_srq_exit0:
int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr)
{
struct ehca_qp *my_qp = container_of(srq, struct ehca_qp, ib_srq);
- struct ehca_pd *my_pd = container_of(srq->pd, struct ehca_pd, ib_pd);
struct ehca_shca *shca = container_of(srq->device, struct ehca_shca,
ib_device);
struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
struct hcp_modify_qp_control_block *qpcb;
- u32 cur_pid = current->tgid;
int ret = 0;
u64 h_ret;
- if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
- my_pd->ownpid != cur_pid) {
- ehca_err(srq->device, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
-
qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
if (!qpcb) {
ehca_err(srq->device, "Out of memory for qpcb "
@@ -1938,7 +1901,7 @@ int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr)
srq_attr->srq_limit = EHCA_BMASK_GET(
MQPCB_CURR_SRQ_LIMIT, qpcb->curr_srq_limit);
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(qpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
query_srq_exit1:
@@ -1954,7 +1917,6 @@ static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
ib_pd);
struct ehca_sport *sport = &shca->sport[my_qp->init_attr.port_num - 1];
- u32 cur_pid = current->tgid;
u32 qp_num = my_qp->real_qp_num;
int ret;
u64 h_ret;
@@ -1969,11 +1931,6 @@ static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
"user space qp_num=%x", qp_num);
return -EINVAL;
}
- if (my_pd->ownpid != cur_pid) {
- ehca_err(dev, "Invalid caller pid=%x ownpid=%x",
- cur_pid, my_pd->ownpid);
- return -EINVAL;
- }
}
if (my_qp->send_cq) {
@@ -1993,6 +1950,9 @@ static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
/* now wait until all pending events have completed */
wait_event(my_qp->wait_completion, !atomic_read(&my_qp->nr_events));
+ /* now wait until all pending events have completed */
+ wait_event(my_qp->wait_completion, !atomic_read(&my_qp->nr_events));
+
h_ret = hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
if (h_ret != H_SUCCESS) {
ehca_err(dev, "hipz_h_destroy_qp() failed h_ret=%li "
diff --git a/drivers/infiniband/hw/ehca/ehca_reqs.c b/drivers/infiniband/hw/ehca/ehca_reqs.c
index fcce580..af6311b 100644
--- a/drivers/infiniband/hw/ehca/ehca_reqs.c
+++ b/drivers/infiniband/hw/ehca/ehca_reqs.c
@@ -81,7 +81,7 @@ static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
recv_wr->sg_list[cnt_ds].length;
}
- if (ehca_debug_level) {
+ if (ehca_debug_level >= 3) {
ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
ipz_rqueue);
ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
@@ -286,7 +286,7 @@ static inline int ehca_write_swqe(struct ehca_qp *qp,
return -EINVAL;
}
- if (ehca_debug_level) {
+ if (ehca_debug_level >= 3) {
ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
}
@@ -434,6 +434,11 @@ int ehca_post_send(struct ib_qp *qp,
int ret = 0;
unsigned long flags;
+ if (unlikely(my_qp->state != IB_QPS_RTS)) {
+ ehca_err(qp->device, "QP not in RTS state qpn=%x", qp->qp_num);
+ return -EINVAL;
+ }
+
/* LOCK the QUEUE */
spin_lock_irqsave(&my_qp->spinlock_s, flags);
@@ -467,13 +472,14 @@ int ehca_post_send(struct ib_qp *qp,
goto post_send_exit0;
}
wqe_cnt++;
- ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
- my_qp, qp->qp_num, wqe_cnt);
} /* eof for cur_send_wr */
post_send_exit0:
iosync(); /* serialize GAL register access */
hipz_update_sqa(my_qp, wqe_cnt);
+ if (unlikely(ret || ehca_debug_level >= 2))
+ ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
+ my_qp, qp->qp_num, wqe_cnt, ret);
my_qp->message_count += wqe_cnt;
spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
return ret;
@@ -533,13 +539,14 @@ static int internal_post_recv(struct ehca_qp *my_qp,
goto post_recv_exit0;
}
wqe_cnt++;
- ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
- my_qp, my_qp->real_qp_num, wqe_cnt);
} /* eof for cur_recv_wr */
post_recv_exit0:
iosync(); /* serialize GAL register access */
hipz_update_rqa(my_qp, wqe_cnt);
+ if (unlikely(ret || ehca_debug_level >= 2))
+ ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
+ my_qp, my_qp->real_qp_num, wqe_cnt, ret);
spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
return ret;
}
@@ -583,16 +590,17 @@ static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
struct ehca_cqe *cqe;
struct ehca_qp *my_qp;
- int cqe_count = 0;
+ int cqe_count = 0, is_error;
poll_cq_one_read_cqe:
cqe = (struct ehca_cqe *)
ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
if (!cqe) {
ret = -EAGAIN;
- ehca_dbg(cq->device, "Completion queue is empty ehca_cq=%p "
- "cq_num=%x ret=%i", my_cq, my_cq->cq_number, ret);
- goto poll_cq_one_exit0;
+ if (ehca_debug_level >= 3)
+ ehca_dbg(cq->device, "Completion queue is empty "
+ "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
+ goto poll_cq_one_exit0;
}
/* prevents loads being reordered across this point */
@@ -622,7 +630,7 @@ poll_cq_one_read_cqe:
ehca_dbg(cq->device,
"Got CQE with purged bit qp_num=%x src_qp=%x",
cqe->local_qp_number, cqe->remote_qp_number);
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
cqe->local_qp_number,
cqe->remote_qp_number);
@@ -635,11 +643,13 @@ poll_cq_one_read_cqe:
}
}
- /* tracing cqe */
- if (unlikely(ehca_debug_level)) {
+ is_error = cqe->status & WC_STATUS_ERROR_BIT;
+
+ /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
+ if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
ehca_dbg(cq->device,
- "Received COMPLETION ehca_cq=%p cq_num=%x -----",
- my_cq, my_cq->cq_number);
+ "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
+ is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
my_cq, my_cq->cq_number);
ehca_dbg(cq->device,
@@ -688,8 +698,9 @@ poll_cq_one_read_cqe:
/* update also queue adder to throw away this entry!!! */
goto poll_cq_one_read_cqe;
}
+
/* eval ib_wc_status */
- if (unlikely(cqe->status & WC_STATUS_ERROR_BIT)) {
+ if (unlikely(is_error)) {
/* complete with errors */
map_ib_wc_status(cqe->status, &wc->status);
wc->vendor_err = wc->status;
@@ -705,14 +716,6 @@ poll_cq_one_read_cqe:
wc->imm_data = cpu_to_be32(cqe->immediate_data);
wc->sl = cqe->service_level;
- if (unlikely(wc->status != IB_WC_SUCCESS))
- ehca_dbg(cq->device,
- "ehca_cq=%p cq_num=%x WARNING unsuccessful cqe "
- "OPType=%x status=%x qp_num=%x src_qp=%x wr_id=%lx "
- "cqe=%p", my_cq, my_cq->cq_number, cqe->optype,
- cqe->status, cqe->local_qp_number,
- cqe->remote_qp_number, cqe->work_request_id, cqe);
-
poll_cq_one_exit0:
if (cqe_count > 0)
hipz_update_feca(my_cq, cqe_count);
diff --git a/drivers/infiniband/hw/ehca/ehca_uverbs.c b/drivers/infiniband/hw/ehca/ehca_uverbs.c
index 5234d6c..e43ed8f 100644
--- a/drivers/infiniband/hw/ehca/ehca_uverbs.c
+++ b/drivers/infiniband/hw/ehca/ehca_uverbs.c
@@ -40,8 +40,6 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#include <asm/current.h>
-
#include "ehca_classes.h"
#include "ehca_iverbs.h"
#include "ehca_mrmw.h"
@@ -213,8 +211,7 @@ static int ehca_mmap_qp(struct vm_area_struct *vma, struct ehca_qp *qp,
break;
case 1: /* qp rqueue_addr */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x rqueue",
- qp->ib_qp.qp_num);
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x rq", qp->ib_qp.qp_num);
ret = ehca_mmap_queue(vma, &qp->ipz_rqueue,
&qp->mm_count_rqueue);
if (unlikely(ret)) {
@@ -226,8 +223,7 @@ static int ehca_mmap_qp(struct vm_area_struct *vma, struct ehca_qp *qp,
break;
case 2: /* qp squeue_addr */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x squeue",
- qp->ib_qp.qp_num);
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x sq", qp->ib_qp.qp_num);
ret = ehca_mmap_queue(vma, &qp->ipz_squeue,
&qp->mm_count_squeue);
if (unlikely(ret)) {
@@ -253,11 +249,9 @@ int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
u32 idr_handle = fileoffset & 0x1FFFFFF;
u32 q_type = (fileoffset >> 27) & 0x1; /* CQ, QP,... */
u32 rsrc_type = (fileoffset >> 25) & 0x3; /* sq,rq,cmnd_window */
- u32 cur_pid = current->tgid;
u32 ret;
struct ehca_cq *cq;
struct ehca_qp *qp;
- struct ehca_pd *pd;
struct ib_uobject *uobject;
switch (q_type) {
@@ -270,13 +264,6 @@ int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
if (!cq)
return -EINVAL;
- if (cq->ownpid != cur_pid) {
- ehca_err(cq->ib_cq.device,
- "Invalid caller pid=%x ownpid=%x",
- cur_pid, cq->ownpid);
- return -ENOMEM;
- }
-
if (!cq->ib_cq.uobject || cq->ib_cq.uobject->context != context)
return -EINVAL;
@@ -298,14 +285,6 @@ int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
if (!qp)
return -EINVAL;
- pd = container_of(qp->ib_qp.pd, struct ehca_pd, ib_pd);
- if (pd->ownpid != cur_pid) {
- ehca_err(qp->ib_qp.device,
- "Invalid caller pid=%x ownpid=%x",
- cur_pid, pd->ownpid);
- return -ENOMEM;
- }
-
uobject = IS_SRQ(qp) ? qp->ib_srq.uobject : qp->ib_qp.uobject;
if (!uobject || uobject->context != context)
return -EINVAL;
diff --git a/drivers/infiniband/hw/ehca/hcp_if.c b/drivers/infiniband/hw/ehca/hcp_if.c
index 7a5496a..415d3a4 100644
--- a/drivers/infiniband/hw/ehca/hcp_if.c
+++ b/drivers/infiniband/hw/ehca/hcp_if.c
@@ -123,8 +123,9 @@ static long ehca_plpar_hcall_norets(unsigned long opcode,
int i, sleep_msecs;
unsigned long flags = 0;
- ehca_gen_dbg("opcode=%lx " HCALL7_REGS_FORMAT,
- opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("opcode=%lx " HCALL7_REGS_FORMAT,
+ opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
for (i = 0; i < 5; i++) {
/* serialize hCalls to work around firmware issue */
@@ -148,7 +149,8 @@ static long ehca_plpar_hcall_norets(unsigned long opcode,
opcode, ret, arg1, arg2, arg3,
arg4, arg5, arg6, arg7);
else
- ehca_gen_dbg("opcode=%lx ret=%li", opcode, ret);
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("opcode=%lx ret=%li", opcode, ret);
return ret;
}
@@ -172,8 +174,10 @@ static long ehca_plpar_hcall9(unsigned long opcode,
int i, sleep_msecs;
unsigned long flags = 0;
- ehca_gen_dbg("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT, opcode,
- arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9);
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT, opcode,
+ arg1, arg2, arg3, arg4, arg5,
+ arg6, arg7, arg8, arg9);
for (i = 0; i < 5; i++) {
/* serialize hCalls to work around firmware issue */
@@ -201,7 +205,7 @@ static long ehca_plpar_hcall9(unsigned long opcode,
ret, outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7],
outs[8]);
- } else
+ } else if (unlikely(ehca_debug_level >= 2))
ehca_gen_dbg("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
ret, outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7],
@@ -381,7 +385,7 @@ u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
r_cb, /* r6 */
0, 0, 0, 0);
- if (ehca_debug_level)
+ if (ehca_debug_level >= 2)
ehca_dmp(query_port_response_block, 64, "response_block");
return ret;
@@ -731,9 +735,6 @@ u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
u64 ret;
u64 outs[PLPAR_HCALL9_BUFSIZE];
- ehca_gen_dbg("kernel PAGE_SIZE=%x access_ctrl=%016x "
- "vaddr=%lx length=%lx",
- (u32)PAGE_SIZE, access_ctrl, vaddr, length);
ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
adapter_handle.handle, /* r4 */
5, /* r5 */
@@ -758,7 +759,7 @@ u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
{
u64 ret;
- if (unlikely(ehca_debug_level >= 2)) {
+ if (unlikely(ehca_debug_level >= 3)) {
if (count > 1) {
u64 *kpage;
int i;
diff --git a/drivers/infiniband/hw/ipath/ipath_7220.h b/drivers/infiniband/hw/ipath/ipath_7220.h
index e918bfd..74fa5cc 100644
--- a/drivers/infiniband/hw/ipath/ipath_7220.h
+++ b/drivers/infiniband/hw/ipath/ipath_7220.h
@@ -39,7 +39,6 @@
* 7220-related files such as ipath_iba7220.c or ipath_sd7220.c.
*/
int ipath_sd7220_presets(struct ipath_devdata *dd);
-int ipath_sd7220_enable_aeq(struct ipath_devdata *dd);
int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset);
int ipath_sd7220_prog_ld(struct ipath_devdata *dd, int sdnum, u8 *img,
int len, int offset);
diff --git a/drivers/infiniband/hw/ipath/ipath_common.h b/drivers/infiniband/hw/ipath/ipath_common.h
index 809d85e..2fa012d 100644
--- a/drivers/infiniband/hw/ipath/ipath_common.h
+++ b/drivers/infiniband/hw/ipath/ipath_common.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
diff --git a/drivers/infiniband/hw/ipath/ipath_diag.c b/drivers/infiniband/hw/ipath/ipath_diag.c
index f0f9e3a..f4eaf13 100644
--- a/drivers/infiniband/hw/ipath/ipath_diag.c
+++ b/drivers/infiniband/hw/ipath/ipath_diag.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c
index c00f5cc..b91d67e 100644
--- a/drivers/infiniband/hw/ipath/ipath_driver.c
+++ b/drivers/infiniband/hw/ipath/ipath_driver.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -74,11 +74,11 @@ module_param_named(debug, ipath_debug, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(debug, "mask for debug prints");
EXPORT_SYMBOL_GPL(ipath_debug);
-unsigned ipath_mtu4096 = 1; /* max 2KB IB mtu by default */
+unsigned ipath_mtu4096 = 1; /* max 4KB IB mtu by default, if supported */
module_param_named(mtu4096, ipath_mtu4096, uint, S_IRUGO);
MODULE_PARM_DESC(mtu4096, "enable MTU of 4096 bytes, if supported");
-static unsigned ipath_hol_timeout_ms = 30000;
+static unsigned ipath_hol_timeout_ms = 13000;
module_param_named(hol_timeout_ms, ipath_hol_timeout_ms, uint, S_IRUGO);
MODULE_PARM_DESC(hol_timeout_ms,
"duration of user app suspension after link failure");
@@ -924,7 +924,8 @@ int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state, int msecs)
return (dd->ipath_flags & state) ? 0 : -ETIMEDOUT;
}
-static void decode_sdma_errs(ipath_err_t err, char *buf, size_t blen)
+static void decode_sdma_errs(struct ipath_devdata *dd, ipath_err_t err,
+ char *buf, size_t blen)
{
static const struct {
ipath_err_t err;
@@ -944,12 +945,16 @@ static void decode_sdma_errs(ipath_err_t err, char *buf, size_t blen)
{ INFINIPATH_E_SDMADISABLED, "SDmaDisabled" },
};
int i;
+ int expected;
size_t bidx = 0;
- for (i = 0; i < ARRAY_SIZE(errs); i++)
- if (err & errs[i].err)
+ for (i = 0; i < ARRAY_SIZE(errs); i++) {
+ expected = (errs[i].err != INFINIPATH_E_SDMADISABLED) ? 0 :
+ test_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
+ if ((err & errs[i].err) && !expected)
bidx += snprintf(buf + bidx, blen - bidx,
"%s ", errs[i].msg);
+ }
}
/*
@@ -958,7 +963,8 @@ static void decode_sdma_errs(ipath_err_t err, char *buf, size_t blen)
* else. Return 1 if "real" errors, otherwise 0 if only packet
* errors, so caller can decide what to print with the string.
*/
-int ipath_decode_err(char *buf, size_t blen, ipath_err_t err)
+int ipath_decode_err(struct ipath_devdata *dd, char *buf, size_t blen,
+ ipath_err_t err)
{
int iserr = 1;
*buf = '\0';
@@ -1041,7 +1047,7 @@ int ipath_decode_err(char *buf, size_t blen, ipath_err_t err)
if (err & INFINIPATH_E_RESET)
strlcat(buf, "reset ", blen);
if (err & INFINIPATH_E_SDMAERRS)
- decode_sdma_errs(err, buf, blen);
+ decode_sdma_errs(dd, err, buf, blen);
if (err & INFINIPATH_E_INVALIDEEPCMD)
strlcat(buf, "invalideepromcmd ", blen);
done:
@@ -1218,7 +1224,7 @@ void ipath_kreceive(struct ipath_portdata *pd)
}
reloop:
- for (last = 0, i = 1; !last; i++) {
+ for (last = 0, i = 1; !last; i += !last) {
hdr = dd->ipath_f_get_msgheader(dd, rhf_addr);
eflags = ipath_hdrget_err_flags(rhf_addr);
etype = ipath_hdrget_rcv_type(rhf_addr);
@@ -1449,6 +1455,41 @@ static void ipath_update_pio_bufs(struct ipath_devdata *dd)
spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
}
+
+/*
+ * used to force update of pioavailshadow if we can't get a pio buffer.
+ * Needed primarily due to exitting freeze mode after recovering
+ * from errors. Done lazily, because it's safer (known to not
+ * be writing pio buffers).
+ */
+static void ipath_reset_availshadow(struct ipath_devdata *dd)
+{
+ int i, im;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ipath_pioavail_lock, flags);
+ for (i = 0; i < dd->ipath_pioavregs; i++) {
+ u64 val, oldval;
+ /* deal with 6110 chip bug on high register #s */
+ im = (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) ?
+ i ^ 1 : i;
+ val = le64_to_cpu(dd->ipath_pioavailregs_dma[im]);
+ /*
+ * busy out the buffers not in the kernel avail list,
+ * without changing the generation bits.
+ */
+ oldval = dd->ipath_pioavailshadow[i];
+ dd->ipath_pioavailshadow[i] = val |
+ ((~dd->ipath_pioavailkernel[i] <<
+ INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT) &
+ 0xaaaaaaaaaaaaaaaaULL); /* All BUSY bits in qword */
+ if (oldval != dd->ipath_pioavailshadow[i])
+ ipath_dbg("shadow[%d] was %Lx, now %lx\n",
+ i, oldval, dd->ipath_pioavailshadow[i]);
+ }
+ spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
+}
+
/**
* ipath_setrcvhdrsize - set the receive header size
* @dd: the infinipath device
@@ -1503,9 +1544,12 @@ static noinline void no_pio_bufs(struct ipath_devdata *dd)
*/
ipath_stats.sps_nopiobufs++;
if (!(++dd->ipath_consec_nopiobuf % 100000)) {
- ipath_dbg("%u pio sends with no bufavail; dmacopy: "
- "%llx %llx %llx %llx; shadow: %lx %lx %lx %lx\n",
+ ipath_force_pio_avail_update(dd); /* at start */
+ ipath_dbg("%u tries no piobufavail ts%lx; dmacopy: "
+ "%llx %llx %llx %llx\n"
+ "ipath shadow: %lx %lx %lx %lx\n",
dd->ipath_consec_nopiobuf,
+ (unsigned long)get_cycles(),
(unsigned long long) le64_to_cpu(dma[0]),
(unsigned long long) le64_to_cpu(dma[1]),
(unsigned long long) le64_to_cpu(dma[2]),
@@ -1517,14 +1561,17 @@ static noinline void no_pio_bufs(struct ipath_devdata *dd)
*/
if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) >
(sizeof(shadow[0]) * 4 * 4))
- ipath_dbg("2nd group: dmacopy: %llx %llx "
- "%llx %llx; shadow: %lx %lx %lx %lx\n",
+ ipath_dbg("2nd group: dmacopy: "
+ "%llx %llx %llx %llx\n"
+ "ipath shadow: %lx %lx %lx %lx\n",
(unsigned long long)le64_to_cpu(dma[4]),
(unsigned long long)le64_to_cpu(dma[5]),
(unsigned long long)le64_to_cpu(dma[6]),
(unsigned long long)le64_to_cpu(dma[7]),
- shadow[4], shadow[5], shadow[6],
- shadow[7]);
+ shadow[4], shadow[5], shadow[6], shadow[7]);
+
+ /* at end, so update likely happened */
+ ipath_reset_availshadow(dd);
}
}
@@ -1648,7 +1695,7 @@ u32 __iomem *ipath_getpiobuf(struct ipath_devdata *dd, u32 plen, u32 *pbufnum)
* Set next starting place. It's just an optimization,
* it doesn't matter who wins on this, so no locking
*/
- if (plen >= IPATH_SMALLBUF_DWORDS)
+ if (plen + 1 >= IPATH_SMALLBUF_DWORDS)
dd->ipath_lastpioindexl = pnum + 1;
else
dd->ipath_lastpioindex = pnum + 1;
@@ -1676,19 +1723,46 @@ void ipath_chg_pioavailkernel(struct ipath_devdata *dd, unsigned start,
unsigned len, int avail)
{
unsigned long flags;
- unsigned end;
+ unsigned end, cnt = 0, next;
/* There are two bits per send buffer (busy and generation) */
start *= 2;
- len *= 2;
- end = start + len;
+ end = start + len * 2;
- /* Set or clear the generation bits. */
spin_lock_irqsave(&ipath_pioavail_lock, flags);
+ /* Set or clear the busy bit in the shadow. */
while (start < end) {
if (avail) {
- __clear_bit(start + INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT,
- dd->ipath_pioavailshadow);
+ unsigned long dma;
+ int i, im;
+ /*
+ * the BUSY bit will never be set, because we disarm
+ * the user buffers before we hand them back to the
+ * kernel. We do have to make sure the generation
+ * bit is set correctly in shadow, since it could
+ * have changed many times while allocated to user.
+ * We can't use the bitmap functions on the full
+ * dma array because it is always little-endian, so
+ * we have to flip to host-order first.
+ * BITS_PER_LONG is slightly wrong, since it's
+ * always 64 bits per register in chip...
+ * We only work on 64 bit kernels, so that's OK.
+ */
+ /* deal with 6110 chip bug on high register #s */
+ i = start / BITS_PER_LONG;
+ im = (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) ?
+ i ^ 1 : i;
+ __clear_bit(INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT
+ + start, dd->ipath_pioavailshadow);
+ dma = (unsigned long) le64_to_cpu(
+ dd->ipath_pioavailregs_dma[im]);
+ if (test_bit((INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
+ + start) % BITS_PER_LONG, &dma))
+ __set_bit(INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
+ + start, dd->ipath_pioavailshadow);
+ else
+ __clear_bit(INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
+ + start, dd->ipath_pioavailshadow);
__set_bit(start, dd->ipath_pioavailkernel);
} else {
__set_bit(start + INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT,
@@ -1697,7 +1771,44 @@ void ipath_chg_pioavailkernel(struct ipath_devdata *dd, unsigned start,
}
start += 2;
}
+
+ if (dd->ipath_pioupd_thresh) {
+ end = 2 * (dd->ipath_piobcnt2k + dd->ipath_piobcnt4k);
+ next = find_first_bit(dd->ipath_pioavailkernel, end);
+ while (next < end) {
+ cnt++;
+ next = find_next_bit(dd->ipath_pioavailkernel, end,
+ next + 1);
+ }
+ }
spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
+
+ /*
+ * When moving buffers from kernel to user, if number assigned to
+ * the user is less than the pio update threshold, and threshold
+ * is supported (cnt was computed > 0), drop the update threshold
+ * so we update at least once per allocated number of buffers.
+ * In any case, if the kernel buffers are less than the threshold,
+ * drop the threshold. We don't bother increasing it, having once
+ * decreased it, since it would typically just cycle back and forth.
+ * If we don't decrease below buffers in use, we can wait a long
+ * time for an update, until some other context uses PIO buffers.
+ */
+ if (!avail && len < cnt)
+ cnt = len;
+ if (cnt < dd->ipath_pioupd_thresh) {
+ dd->ipath_pioupd_thresh = cnt;
+ ipath_dbg("Decreased pio update threshold to %u\n",
+ dd->ipath_pioupd_thresh);
+ spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
+ dd->ipath_sendctrl &= ~(INFINIPATH_S_UPDTHRESH_MASK
+ << INFINIPATH_S_UPDTHRESH_SHIFT);
+ dd->ipath_sendctrl |= dd->ipath_pioupd_thresh
+ << INFINIPATH_S_UPDTHRESH_SHIFT;
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
+ dd->ipath_sendctrl);
+ spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
+ }
}
/**
@@ -1807,12 +1918,23 @@ void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl)
goto bail;
}
/*
- * If we have SDMA, we have to kick off the abort state machine
- * and make sure we aren't already aborting.
+ * If we have SDMA, and it's not disabled, we have to kick off the
+ * abort state machine, provided we aren't already aborting.
+ * If we are in the process of aborting SDMA (!DISABLED, but ABORTING),
+ * we skip the rest of this routine. It is already "in progress"
*/
- if ((dd->ipath_flags & IPATH_HAS_SEND_DMA) &&
- (test_and_set_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status)))
- goto bail;
+ if (dd->ipath_flags & IPATH_HAS_SEND_DMA) {
+ int skip_cancel;
+ u64 *statp = &dd->ipath_sdma_status;
+
+ spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
+ skip_cancel =
+ test_and_set_bit(IPATH_SDMA_ABORTING, statp)
+ && !test_bit(IPATH_SDMA_DISABLED, statp);
+ spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
+ if (skip_cancel)
+ goto bail;
+ }
ipath_dbg("Cancelling all in-progress send buffers\n");
@@ -1839,6 +1961,9 @@ void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl)
ipath_disarm_piobufs(dd, 0,
dd->ipath_piobcnt2k + dd->ipath_piobcnt4k);
+ if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
+ set_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
+
if (restore_sendctrl) {
/* else done by caller later if needed */
spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
@@ -1852,12 +1977,12 @@ void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl)
}
if ((dd->ipath_flags & IPATH_HAS_SEND_DMA) &&
+ !test_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status) &&
test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status)) {
spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
/* only wait so long for intr */
dd->ipath_sdma_abort_intr_timeout = jiffies + HZ;
dd->ipath_sdma_reset_wait = 200;
- __set_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
tasklet_hi_schedule(&dd->ipath_sdma_abort_task);
spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
@@ -1906,6 +2031,24 @@ static void ipath_set_ib_lstate(struct ipath_devdata *dd, int linkcmd,
[INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED",
[INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE"
};
+ if (linitcmd == INFINIPATH_IBCC_LINKINITCMD_DISABLE) {
+ /*
+ * If we are told to disable, note that so link-recovery
+ * code does not attempt to bring us back up.
+ */
+ preempt_disable();
+ dd->ipath_flags |= IPATH_IB_LINK_DISABLED;
+ preempt_enable();
+ } else if (linitcmd) {
+ /*
+ * Any other linkinitcmd will lead to LINKDOWN< and then
+ * to INIT (if all is well), so clear flag to let ink-recovery
+ * code attempt to bring us back up.
+ */
+ preempt_disable();
+ dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED;
+ preempt_enable();
+ }
mod_wd = (linkcmd << dd->ibcc_lc_shift) |
(linitcmd << INFINIPATH_IBCC_LINKINITCMD_SHIFT);
@@ -2249,7 +2392,7 @@ void ipath_shutdown_device(struct ipath_devdata *dd)
ipath_dbg("Shutting down the device\n");
- ipath_hol_up(dd);
+ ipath_hol_up(dd); /* make sure user processes aren't suspended */
dd->ipath_flags |= IPATH_LINKUNK;
dd->ipath_flags &= ~(IPATH_INITTED | IPATH_LINKDOWN |
@@ -2592,48 +2735,76 @@ int ipath_signal_procs(struct ipath_devdata *dd, int sig)
return any;
}
+static void ipath_hol_signal_down(struct ipath_devdata *dd)
+{
+ if (ipath_signal_procs(dd, SIGSTOP))
+ ipath_dbg("Stopped some processes\n");
+ ipath_cancel_sends(dd, 1);
+}
+
+
+static void ipath_hol_signal_up(struct ipath_devdata *dd)
+{
+ if (ipath_signal_procs(dd, SIGCONT))
+ ipath_dbg("Continued some processes\n");
+}
+
+/*
+ * link is down, stop any users processes, and flush pending sends
+ * to prevent HoL blocking, then start the HoL timer that
+ * periodically continues, then stop procs, so they can detect
+ * link down if they want, and do something about it.
+ * Timer may already be running, so use __mod_timer, not add_timer.
+ */
void ipath_hol_down(struct ipath_devdata *dd)
{
- del_timer_sync(&dd->ipath_hol_timer);
- dd->ipath_hol_state = IPATH_HOL_DOWNCONT;
- ipath_hol_event((unsigned long)dd);
+ dd->ipath_hol_state = IPATH_HOL_DOWN;
+ ipath_hol_signal_down(dd);
+ dd->ipath_hol_next = IPATH_HOL_DOWNCONT;
+ dd->ipath_hol_timer.expires = jiffies +
+ msecs_to_jiffies(ipath_hol_timeout_ms);
+ __mod_timer(&dd->ipath_hol_timer, dd->ipath_hol_timer.expires);
}
+/*
+ * link is up, continue any user processes, and ensure timer
+ * is a nop, if running. Let timer keep running, if set; it
+ * will nop when it sees the link is up
+ */
void ipath_hol_up(struct ipath_devdata *dd)
{
- del_timer_sync(&dd->ipath_hol_timer);
+ ipath_hol_signal_up(dd);
dd->ipath_hol_state = IPATH_HOL_UP;
- ipath_hol_event((unsigned long)dd);
}
+/*
+ * toggle the running/not running state of user proceses
+ * to prevent HoL blocking on chip resources, but still allow
+ * user processes to do link down special case handling.
+ * Should only be called via the timer
+ */
void ipath_hol_event(unsigned long opaque)
{
struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
- unsigned oldstate = dd->ipath_hol_state;
- unsigned newstate = IPATH_HOL_UP;
-
- if (!(dd->ipath_flags & IPATH_LINKACTIVE)) {
- if (oldstate == IPATH_HOL_DOWNSTOP)
- newstate = IPATH_HOL_DOWNCONT;
- else
- newstate = IPATH_HOL_DOWNSTOP;
- }
- if (newstate == IPATH_HOL_DOWNSTOP) {
- if (ipath_signal_procs(dd, SIGSTOP))
- ipath_dbg("Stopped some processes\n");
- ipath_cancel_sends(dd, 1);
- } else
- if (ipath_signal_procs(dd, SIGCONT))
- ipath_dbg("Continued some processes\n");
-
- if ((oldstate != newstate) || (newstate != IPATH_HOL_UP)) {
+ if (dd->ipath_hol_next == IPATH_HOL_DOWNSTOP
+ && dd->ipath_hol_state != IPATH_HOL_UP) {
+ dd->ipath_hol_next = IPATH_HOL_DOWNCONT;
+ ipath_dbg("Stopping processes\n");
+ ipath_hol_signal_down(dd);
+ } else { /* may do "extra" if also in ipath_hol_up() */
+ dd->ipath_hol_next = IPATH_HOL_DOWNSTOP;
+ ipath_dbg("Continuing processes\n");
+ ipath_hol_signal_up(dd);
+ }
+ if (dd->ipath_hol_state == IPATH_HOL_UP)
+ ipath_dbg("link's up, don't resched timer\n");
+ else {
dd->ipath_hol_timer.expires = jiffies +
msecs_to_jiffies(ipath_hol_timeout_ms);
- add_timer(&dd->ipath_hol_timer);
+ __mod_timer(&dd->ipath_hol_timer,
+ dd->ipath_hol_timer.expires);
}
-
- dd->ipath_hol_state = newstate;
}
int ipath_set_rx_pol_inv(struct ipath_devdata *dd, u8 new_pol_inv)
diff --git a/drivers/infiniband/hw/ipath/ipath_eeprom.c b/drivers/infiniband/hw/ipath/ipath_eeprom.c
index 810b5fe..899f156 100644
--- a/drivers/infiniband/hw/ipath/ipath_eeprom.c
+++ b/drivers/infiniband/hw/ipath/ipath_eeprom.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c
index abadeef..dc61e15 100644
--- a/drivers/infiniband/hw/ipath/ipath_file_ops.c
+++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -173,46 +173,25 @@ static int ipath_get_base_info(struct file *fp,
(void *) dd->ipath_statusp -
(void *) dd->ipath_pioavailregs_dma;
if (!shared) {
- kinfo->spi_piocnt = dd->ipath_pbufsport;
+ kinfo->spi_piocnt = pd->port_piocnt;
kinfo->spi_piobufbase = (u64) pd->port_piobufs;
kinfo->__spi_uregbase = (u64) dd->ipath_uregbase +
dd->ipath_ureg_align * pd->port_port;
} else if (master) {
- kinfo->spi_piocnt = (dd->ipath_pbufsport / subport_cnt) +
- (dd->ipath_pbufsport % subport_cnt);
+ kinfo->spi_piocnt = (pd->port_piocnt / subport_cnt) +
+ (pd->port_piocnt % subport_cnt);
/* Master's PIO buffers are after all the slave's */
kinfo->spi_piobufbase = (u64) pd->port_piobufs +
dd->ipath_palign *
- (dd->ipath_pbufsport - kinfo->spi_piocnt);
+ (pd->port_piocnt - kinfo->spi_piocnt);
} else {
unsigned slave = subport_fp(fp) - 1;
- kinfo->spi_piocnt = dd->ipath_pbufsport / subport_cnt;
+ kinfo->spi_piocnt = pd->port_piocnt / subport_cnt;
kinfo->spi_piobufbase = (u64) pd->port_piobufs +
dd->ipath_palign * kinfo->spi_piocnt * slave;
}
- /*
- * Set the PIO avail update threshold to no larger
- * than the number of buffers per process. Note that
- * we decrease it here, but won't ever increase it.
- */
- if (dd->ipath_pioupd_thresh &&
- kinfo->spi_piocnt < dd->ipath_pioupd_thresh) {
- unsigned long flags;
- dd->ipath_pioupd_thresh = kinfo->spi_piocnt;
- ipath_dbg("Decreased pio update threshold to %u\n",
- dd->ipath_pioupd_thresh);
- spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
- dd->ipath_sendctrl &= ~(INFINIPATH_S_UPDTHRESH_MASK
- << INFINIPATH_S_UPDTHRESH_SHIFT);
- dd->ipath_sendctrl |= dd->ipath_pioupd_thresh
- << INFINIPATH_S_UPDTHRESH_SHIFT;
- ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
- dd->ipath_sendctrl);
- spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
- }
-
if (shared) {
kinfo->spi_port_uregbase = (u64) dd->ipath_uregbase +
dd->ipath_ureg_align * pd->port_port;
@@ -1317,19 +1296,19 @@ static int ipath_mmap(struct file *fp, struct vm_area_struct *vma)
ureg = dd->ipath_uregbase + dd->ipath_ureg_align * pd->port_port;
if (!pd->port_subport_cnt) {
/* port is not shared */
- piocnt = dd->ipath_pbufsport;
+ piocnt = pd->port_piocnt;
piobufs = pd->port_piobufs;
} else if (!subport_fp(fp)) {
/* caller is the master */
- piocnt = (dd->ipath_pbufsport / pd->port_subport_cnt) +
- (dd->ipath_pbufsport % pd->port_subport_cnt);
+ piocnt = (pd->port_piocnt / pd->port_subport_cnt) +
+ (pd->port_piocnt % pd->port_subport_cnt);
piobufs = pd->port_piobufs +
- dd->ipath_palign * (dd->ipath_pbufsport - piocnt);
+ dd->ipath_palign * (pd->port_piocnt - piocnt);
} else {
unsigned slave = subport_fp(fp) - 1;
/* caller is a slave */
- piocnt = dd->ipath_pbufsport / pd->port_subport_cnt;
+ piocnt = pd->port_piocnt / pd->port_subport_cnt;
piobufs = pd->port_piobufs + dd->ipath_palign * piocnt * slave;
}
@@ -1641,9 +1620,6 @@ static int try_alloc_port(struct ipath_devdata *dd, int port,
port_fp(fp) = pd;
pd->port_pid = current->pid;
strncpy(pd->port_comm, current->comm, sizeof(pd->port_comm));
- ipath_chg_pioavailkernel(dd,
- dd->ipath_pbufsport * (pd->port_port - 1),
- dd->ipath_pbufsport, 0);
ipath_stats.sps_ports++;
ret = 0;
} else
@@ -1806,7 +1782,7 @@ static int find_shared_port(struct file *fp,
for (ndev = 0; ndev < devmax; ndev++) {
struct ipath_devdata *dd = ipath_lookup(ndev);
- if (!dd)
+ if (!usable(dd))
continue;
for (i = 1; i < dd->ipath_cfgports; i++) {
struct ipath_portdata *pd = dd->ipath_pd[i];
@@ -1946,11 +1922,25 @@ static int ipath_do_user_init(struct file *fp,
/* for now we do nothing with rcvhdrcnt: uinfo->spu_rcvhdrcnt */
+ /* some ports may get extra buffers, calculate that here */
+ if (pd->port_port <= dd->ipath_ports_extrabuf)
+ pd->port_piocnt = dd->ipath_pbufsport + 1;
+ else
+ pd->port_piocnt = dd->ipath_pbufsport;
+
/* for right now, kernel piobufs are at end, so port 1 is at 0 */
+ if (pd->port_port <= dd->ipath_ports_extrabuf)
+ pd->port_pio_base = (dd->ipath_pbufsport + 1)
+ * (pd->port_port - 1);
+ else
+ pd->port_pio_base = dd->ipath_ports_extrabuf +
+ dd->ipath_pbufsport * (pd->port_port - 1);
pd->port_piobufs = dd->ipath_piobufbase +
- dd->ipath_pbufsport * (pd->port_port - 1) * dd->ipath_palign;
- ipath_cdbg(VERBOSE, "Set base of piobufs for port %u to 0x%x\n",
- pd->port_port, pd->port_piobufs);
+ pd->port_pio_base * dd->ipath_palign;
+ ipath_cdbg(VERBOSE, "piobuf base for port %u is 0x%x, piocnt %u,"
+ " first pio %u\n", pd->port_port, pd->port_piobufs,
+ pd->port_piocnt, pd->port_pio_base);
+ ipath_chg_pioavailkernel(dd, pd->port_pio_base, pd->port_piocnt, 0);
/*
* Now allocate the rcvhdr Q and eager TIDs; skip the TID
@@ -2115,7 +2105,6 @@ static int ipath_close(struct inode *in, struct file *fp)
}
if (dd->ipath_kregbase) {
- int i;
/* atomically clear receive enable port and intr avail. */
clear_bit(dd->ipath_r_portenable_shift + port,
&dd->ipath_rcvctrl);
@@ -2144,9 +2133,9 @@ static int ipath_close(struct inode *in, struct file *fp)
ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
pd->port_port, dd->ipath_dummy_hdrq_phys);
- i = dd->ipath_pbufsport * (port - 1);
- ipath_disarm_piobufs(dd, i, dd->ipath_pbufsport);
- ipath_chg_pioavailkernel(dd, i, dd->ipath_pbufsport, 1);
+ ipath_disarm_piobufs(dd, pd->port_pio_base, pd->port_piocnt);
+ ipath_chg_pioavailkernel(dd, pd->port_pio_base,
+ pd->port_piocnt, 1);
dd->ipath_f_clear_tids(dd, pd->port_port);
diff --git a/drivers/infiniband/hw/ipath/ipath_iba6110.c b/drivers/infiniband/hw/ipath/ipath_iba6110.c
index 3e59301..6559c92 100644
--- a/drivers/infiniband/hw/ipath/ipath_iba6110.c
+++ b/drivers/infiniband/hw/ipath/ipath_iba6110.c
@@ -848,7 +848,7 @@ static void slave_or_pri_blk(struct ipath_devdata *dd, struct pci_dev *pdev,
/*
* now write them back to clear the error.
*/
- pci_write_config_byte(pdev, link_off,
+ pci_write_config_word(pdev, link_off,
linkctrl & (0xf << 8));
}
}
diff --git a/drivers/infiniband/hw/ipath/ipath_iba6120.c b/drivers/infiniband/hw/ipath/ipath_iba6120.c
index d83eece..790b8f7 100644
--- a/drivers/infiniband/hw/ipath/ipath_iba6120.c
+++ b/drivers/infiniband/hw/ipath/ipath_iba6120.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
diff --git a/drivers/infiniband/hw/ipath/ipath_iba7220.c b/drivers/infiniband/hw/ipath/ipath_iba7220.c
index ce1e421..f16cf9c 100644
--- a/drivers/infiniband/hw/ipath/ipath_iba7220.c
+++ b/drivers/infiniband/hw/ipath/ipath_iba7220.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -600,7 +600,7 @@ static void ipath_7220_txe_recover(struct ipath_devdata *dd)
dev_info(&dd->pcidev->dev,
"Recovering from TXE PIO parity error\n");
- ipath_disarm_senderrbufs(dd, 1);
+ ipath_disarm_senderrbufs(dd);
}
@@ -674,13 +674,14 @@ static void ipath_7220_handle_hwerrors(struct ipath_devdata *dd, char *msg,
"%llx set\n", (unsigned long long)
(hwerrs & ~dd->ipath_hwe_bitsextant));
+ if (hwerrs & INFINIPATH_HWE_IB_UC_MEMORYPARITYERR)
+ ipath_sd7220_clr_ibpar(dd);
+
ctrl = ipath_read_kreg32(dd, dd->ipath_kregs->kr_control);
if ((ctrl & INFINIPATH_C_FREEZEMODE) && !ipath_diag_inuse) {
/*
- * Parity errors in send memory are recoverable,
- * just cancel the send (if indicated in * sendbuffererror),
- * count the occurrence, unfreeze (if no other handled
- * hardware error bits are set), and continue.
+ * Parity errors in send memory are recoverable by h/w
+ * just do housekeeping, exit freeze mode and continue.
*/
if (hwerrs & ((INFINIPATH_HWE_TXEMEMPARITYERR_PIOBUF |
INFINIPATH_HWE_TXEMEMPARITYERR_PIOPBC)
@@ -689,13 +690,6 @@ static void ipath_7220_handle_hwerrors(struct ipath_devdata *dd, char *msg,
hwerrs &= ~((INFINIPATH_HWE_TXEMEMPARITYERR_PIOBUF |
INFINIPATH_HWE_TXEMEMPARITYERR_PIOPBC)
<< INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT);
- if (!hwerrs) {
- /* else leave in freeze mode */
- ipath_write_kreg(dd,
- dd->ipath_kregs->kr_control,
- dd->ipath_control);
- goto bail;
- }
}
if (hwerrs) {
/*
@@ -725,8 +719,8 @@ static void ipath_7220_handle_hwerrors(struct ipath_devdata *dd, char *msg,
*dd->ipath_statusp |= IPATH_STATUS_HWERROR;
dd->ipath_flags &= ~IPATH_INITTED;
} else {
- ipath_dbg("Clearing freezemode on ignored hardware "
- "error\n");
+ ipath_dbg("Clearing freezemode on ignored or "
+ "recovered hardware error\n");
ipath_clear_freeze(dd);
}
}
@@ -875,11 +869,19 @@ static int ipath_7220_boardname(struct ipath_devdata *dd, char *name,
snprintf(name, namelen, "%s", n);
}
- if (dd->ipath_majrev < 4 || !dd->ipath_minrev || dd->ipath_minrev > 2) {
+ if (dd->ipath_majrev != 5 || !dd->ipath_minrev ||
+ dd->ipath_minrev > 2) {
ipath_dev_err(dd, "Unsupported InfiniPath hardware "
"revision %u.%u!\n",
dd->ipath_majrev, dd->ipath_minrev);
ret = 1;
+ } else if (dd->ipath_minrev == 1 &&
+ !(dd->ipath_flags & IPATH_INITTED)) {
+ /* Rev1 chips are prototype. Complain at init, but allow use */
+ ipath_dev_err(dd, "Unsupported hardware "
+ "revision %u.%u, Contact support@qlogic.com\n",
+ dd->ipath_majrev, dd->ipath_minrev);
+ ret = 0;
} else
ret = 0;
@@ -923,6 +925,7 @@ static void ipath_7220_init_hwerrors(struct ipath_devdata *dd)
if (dd->ipath_minrev == 1)
val &= ~(1ULL << 42); /* TXE LaunchFIFO Parity rev1 issue */
+ val &= ~INFINIPATH_HWE_IB_UC_MEMORYPARITYERR;
dd->ipath_hwerrmask = val;
/*
@@ -1030,9 +1033,13 @@ static int ipath_7220_bringup_serdes(struct ipath_devdata *dd)
val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
prev_val = val;
+ val |= INFINIPATH_XGXS_FC_SAFE;
+ if (val != prev_val) {
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
+ ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
+ }
if (val & INFINIPATH_XGXS_RESET)
val &= ~INFINIPATH_XGXS_RESET;
- val |= INFINIPATH_XGXS_FC_SAFE;
if (val != prev_val)
ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
@@ -1089,10 +1096,15 @@ static void ipath_7220_config_jint(struct ipath_devdata *dd,
*/
static void ipath_7220_quiet_serdes(struct ipath_devdata *dd)
{
+ u64 val;
dd->ipath_flags &= ~IPATH_IB_AUTONEG_INPROG;
wake_up(&dd->ipath_autoneg_wait);
cancel_delayed_work(&dd->ipath_autoneg_work);
flush_scheduled_work();
+ ipath_shutdown_relock_poll(dd);
+ val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
+ val |= INFINIPATH_XGXS_RESET;
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
}
static int ipath_7220_intconfig(struct ipath_devdata *dd)
@@ -1953,7 +1965,7 @@ static void ipath_7220_config_ports(struct ipath_devdata *dd, ushort cfgports)
dd->ipath_rcvctrl);
dd->ipath_p0_rcvegrcnt = 2048; /* always */
if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
- dd->ipath_pioreserved = 1; /* reserve a buffer */
+ dd->ipath_pioreserved = 3; /* kpiobufs used for PIO */
}
@@ -2305,7 +2317,7 @@ static void try_auto_neg(struct ipath_devdata *dd)
dd->ipath_flags |= IPATH_IB_AUTONEG_INPROG;
ipath_autoneg_send(dd, 0);
set_speed_fast(dd, IPATH_IB_DDR);
- ipath_set_linkstate(dd, IPATH_IB_LINKDOWN);
+ ipath_toggle_rclkrls(dd);
/* 2 msec is minimum length of a poll cycle */
schedule_delayed_work(&dd->ipath_autoneg_work,
msecs_to_jiffies(2));
@@ -2338,9 +2350,11 @@ static int ipath_7220_ib_updown(struct ipath_devdata *dd, int ibup, u64 ibcs)
IPATH_IB_AUTONEG_INPROG)))
set_speed_fast(dd, dd->ipath_link_speed_enabled);
if (!(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG)) {
- ipath_cdbg(VERBOSE, "Disabling AEQ\n");
+ ipath_cdbg(VERBOSE, "Setting RXEQ defaults\n");
ipath_sd7220_presets(dd);
}
+ /* this might better in ipath_sd7220_presets() */
+ ipath_set_relock_poll(dd, ibup);
} else {
if (ipath_compat_ddr_negotiate &&
!(dd->ipath_flags & (IPATH_IB_AUTONEG_FAILED |
@@ -2362,7 +2376,7 @@ static int ipath_7220_ib_updown(struct ipath_devdata *dd, int ibup, u64 ibcs)
ipath_autoneg_send(dd, 1);
set_speed_fast(dd, IPATH_IB_DDR);
udelay(2);
- ipath_set_linkstate(dd, IPATH_IB_LINKDOWN);
+ ipath_toggle_rclkrls(dd);
ret = 1; /* no other IB status change processing */
} else {
if ((dd->ipath_flags & IPATH_IB_AUTONEG_INPROG) &&
@@ -2423,8 +2437,7 @@ static int ipath_7220_ib_updown(struct ipath_devdata *dd, int ibup, u64 ibcs)
[(ibcs >> IBA7220_IBCS_LINKSPEED_SHIFT) & 1]
[(ibcs >> IBA7220_IBCS_LINKWIDTH_SHIFT) & 1];
- ipath_cdbg(VERBOSE, "enabling AEQ\n");
- ipath_sd7220_enable_aeq(dd);
+ ipath_set_relock_poll(dd, ibup);
}
}
@@ -2443,7 +2456,7 @@ static void autoneg_work(struct work_struct *work)
{
struct ipath_devdata *dd;
u64 startms;
- u32 ltstate, lastlts, i;
+ u32 lastlts, i;
dd = container_of(work, struct ipath_devdata,
ipath_autoneg_work.work);
@@ -2455,8 +2468,6 @@ static void autoneg_work(struct work_struct *work)
* few hundred usec, since we scheduled ourselves for 2msec.
*/
for (i = 0; i < 25; i++) {
- ltstate = ipath_ib_linktrstate(dd, ipath_read_kreg64(dd,
- dd->ipath_kregs->kr_ibcstatus));
lastlts = ipath_ib_linktrstate(dd, dd->ipath_lastibcstat);
if (lastlts == INFINIPATH_IBCS_LT_STATE_POLLQUIET) {
ipath_set_linkstate(dd, IPATH_IB_LINKDOWN_DISABLE);
@@ -2474,7 +2485,7 @@ static void autoneg_work(struct work_struct *work)
msecs_to_jiffies(90)))
goto done;
- ipath_set_linkstate(dd, IPATH_IB_LINKDOWN);
+ ipath_toggle_rclkrls(dd);
/* we expect this to timeout */
if (wait_event_timeout(dd->ipath_autoneg_wait,
@@ -2483,6 +2494,7 @@ static void autoneg_work(struct work_struct *work)
goto done;
set_speed_fast(dd, IPATH_IB_SDR);
+ ipath_toggle_rclkrls(dd);
/*
* wait up to 250 msec for link to train and get to INIT at DDR;
@@ -2504,16 +2516,6 @@ done:
dd->ipath_autoneg_tries = 0;
}
set_speed_fast(dd, dd->ipath_link_speed_enabled);
-
- /*
- * we may be in a stuck state, where we won't get another
- * link transition. If so forcibly bounce the link.
- */
- if (lastlts >= INFINIPATH_IBCS_LT_STATE_CFGDEBOUNCE) {
- ipath_dbg("Bouncing IB link, lastibcstat %LX\n",
- dd->ipath_lastibcstat);
- ipath_set_linkstate(dd, IPATH_IB_LINKDOWN);
- }
}
}
diff --git a/drivers/infiniband/hw/ipath/ipath_init_chip.c b/drivers/infiniband/hw/ipath/ipath_init_chip.c
index a9cceb8..edd5c92 100644
--- a/drivers/infiniband/hw/ipath/ipath_init_chip.c
+++ b/drivers/infiniband/hw/ipath/ipath_init_chip.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -41,7 +41,7 @@
/*
* min buffers we want to have per port, after driver
*/
-#define IPATH_MIN_USER_PORT_BUFCNT 8
+#define IPATH_MIN_USER_PORT_BUFCNT 7
/*
* Number of ports we are configured to use (to allow for more pio
@@ -54,13 +54,9 @@ MODULE_PARM_DESC(cfgports, "Set max number of ports to use");
/*
* Number of buffers reserved for driver (verbs and layered drivers.)
- * Reserved at end of buffer list. Initialized based on
- * number of PIO buffers if not set via module interface.
+ * Initialized based on number of PIO buffers if not set via module interface.
* The problem with this is that it's global, but we'll use different
- * numbers for different chip types. So the default value is not
- * very useful. I've redefined it for the 1.3 release so that it's
- * zero unless set by the user to something else, in which case we
- * try to respect it.
+ * numbers for different chip types.
*/
static ushort ipath_kpiobufs;
@@ -180,10 +176,13 @@ static int bringup_link(struct ipath_devdata *dd)
/*
* Want to start out with both LINKCMD and LINKINITCMD in NOP
* (0 and 0). Don't put linkinitcmd in ipath_ibcctrl, want that
- * to stay a NOP
+ * to stay a NOP. Flag that we are disabled, for the (unlikely)
+ * case that some recovery path is trying to bring the link up
+ * before we are ready.
*/
ibc |= INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
INFINIPATH_IBCC_LINKINITCMD_SHIFT;
+ dd->ipath_flags |= IPATH_IB_LINK_DISABLED;
ipath_cdbg(VERBOSE, "Writing 0x%llx to ibcctrl\n",
(unsigned long long) ibc);
ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, ibc);
@@ -545,9 +544,12 @@ static void enable_chip(struct ipath_devdata *dd, int reinit)
tmp = dd->ipath_pioavailregs_dma[i ^ 1];
else
tmp = dd->ipath_pioavailregs_dma[i];
- dd->ipath_pioavailshadow[i] = le64_to_cpu(tmp) |
- (~dd->ipath_pioavailkernel[i] <<
- INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT);
+ /*
+ * don't need to worry about ipath_pioavailkernel here
+ * because we will call ipath_chg_pioavailkernel() later
+ * in initialization, to busy out buffers as needed
+ */
+ dd->ipath_pioavailshadow[i] = le64_to_cpu(tmp);
}
/* can get counters, stats, etc. */
dd->ipath_flags |= IPATH_PRESENT;
@@ -712,12 +714,11 @@ static void verify_interrupt(unsigned long opaque)
int ipath_init_chip(struct ipath_devdata *dd, int reinit)
{
int ret = 0;
- u32 val32, kpiobufs;
+ u32 kpiobufs, defkbufs;
u32 piobufs, uports;
u64 val;
struct ipath_portdata *pd;
gfp_t gfp_flags = GFP_USER | __GFP_COMP;
- unsigned long flags;
ret = init_housekeeping(dd, reinit);
if (ret)
@@ -757,56 +758,46 @@ int ipath_init_chip(struct ipath_devdata *dd, int reinit)
dd->ipath_pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2)
/ (sizeof(u64) * BITS_PER_BYTE / 2);
uports = dd->ipath_cfgports ? dd->ipath_cfgports - 1 : 0;
- if (ipath_kpiobufs == 0) {
- /* not set by user (this is default) */
- if (piobufs > 144)
- kpiobufs = 32;
- else
- kpiobufs = 16;
- }
+ if (piobufs > 144)
+ defkbufs = 32 + dd->ipath_pioreserved;
else
- kpiobufs = ipath_kpiobufs;
+ defkbufs = 16 + dd->ipath_pioreserved;
- if (kpiobufs + (uports * IPATH_MIN_USER_PORT_BUFCNT) > piobufs) {
+ if (ipath_kpiobufs && (ipath_kpiobufs +
+ (uports * IPATH_MIN_USER_PORT_BUFCNT)) > piobufs) {
int i = (int) piobufs -
(int) (uports * IPATH_MIN_USER_PORT_BUFCNT);
if (i < 1)
i = 1;
dev_info(&dd->pcidev->dev, "Allocating %d PIO bufs of "
"%d for kernel leaves too few for %d user ports "
- "(%d each); using %u\n", kpiobufs,
+ "(%d each); using %u\n", ipath_kpiobufs,
piobufs, uports, IPATH_MIN_USER_PORT_BUFCNT, i);
/*
* shouldn't change ipath_kpiobufs, because could be
* different for different devices...
*/
kpiobufs = i;
- }
+ } else if (ipath_kpiobufs)
+ kpiobufs = ipath_kpiobufs;
+ else
+ kpiobufs = defkbufs;
dd->ipath_lastport_piobuf = piobufs - kpiobufs;
dd->ipath_pbufsport =
uports ? dd->ipath_lastport_piobuf / uports : 0;
- val32 = dd->ipath_lastport_piobuf - (dd->ipath_pbufsport * uports);
- if (val32 > 0) {
- ipath_dbg("allocating %u pbufs/port leaves %u unused, "
- "add to kernel\n", dd->ipath_pbufsport, val32);
- dd->ipath_lastport_piobuf -= val32;
- kpiobufs += val32;
- ipath_dbg("%u pbufs/port leaves %u unused, add to kernel\n",
- dd->ipath_pbufsport, val32);
- }
+ /* if not an even divisor, some user ports get extra buffers */
+ dd->ipath_ports_extrabuf = dd->ipath_lastport_piobuf -
+ (dd->ipath_pbufsport * uports);
+ if (dd->ipath_ports_extrabuf)
+ ipath_dbg("%u pbufs/port leaves some unused, add 1 buffer to "
+ "ports <= %u\n", dd->ipath_pbufsport,
+ dd->ipath_ports_extrabuf);
dd->ipath_lastpioindex = 0;
dd->ipath_lastpioindexl = dd->ipath_lastport_piobuf;
- ipath_chg_pioavailkernel(dd, 0, piobufs, 1);
+ /* ipath_pioavailshadow initialized earlier */
ipath_cdbg(VERBOSE, "%d PIO bufs for kernel out of %d total %u "
"each for %u user ports\n", kpiobufs,
piobufs, dd->ipath_pbufsport, uports);
- if (dd->ipath_pioupd_thresh) {
- if (dd->ipath_pbufsport < dd->ipath_pioupd_thresh)
- dd->ipath_pioupd_thresh = dd->ipath_pbufsport;
- if (kpiobufs < dd->ipath_pioupd_thresh)
- dd->ipath_pioupd_thresh = kpiobufs;
- }
-
ret = dd->ipath_f_early_init(dd);
if (ret) {
ipath_dev_err(dd, "Early initialization failure\n");
@@ -814,13 +805,6 @@ int ipath_init_chip(struct ipath_devdata *dd, int reinit)
}
/*
- * Cancel any possible active sends from early driver load.
- * Follows early_init because some chips have to initialize
- * PIO buffers in early_init to avoid false parity errors.
- */
- ipath_cancel_sends(dd, 0);
-
- /*
* Early_init sets rcvhdrentsize and rcvhdrsize, so this must be
* done after early_init.
*/
@@ -840,6 +824,7 @@ int ipath_init_chip(struct ipath_devdata *dd, int reinit)
ipath_write_kreg(dd, dd->ipath_kregs->kr_sendpioavailaddr,
dd->ipath_pioavailregs_phys);
+
/*
* this is to detect s/w errors, which the h/w works around by
* ignoring the low 6 bits of address, if it wasn't aligned.
@@ -866,12 +851,6 @@ int ipath_init_chip(struct ipath_devdata *dd, int reinit)
~0ULL&~INFINIPATH_HWE_MEMBISTFAILED);
ipath_write_kreg(dd, dd->ipath_kregs->kr_control, 0ULL);
- spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
- dd->ipath_sendctrl = INFINIPATH_S_PIOENABLE;
- ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
- ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
- spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
-
/*
* before error clears, since we expect serdes pll errors during
* this, the first time after reset
@@ -944,6 +923,19 @@ int ipath_init_chip(struct ipath_devdata *dd, int reinit)
else
enable_chip(dd, reinit);
+ /* after enable_chip, so pioavailshadow setup */
+ ipath_chg_pioavailkernel(dd, 0, piobufs, 1);
+
+ /*
+ * Cancel any possible active sends from early driver load.
+ * Follows early_init because some chips have to initialize
+ * PIO buffers in early_init to avoid false parity errors.
+ * After enable and ipath_chg_pioavailkernel so we can safely
+ * enable pioavail updates and PIOENABLE; packets are now
+ * ready to go out.
+ */
+ ipath_cancel_sends(dd, 1);
+
if (!reinit) {
/*
* Used when we close a port, for DMA already in flight
diff --git a/drivers/infiniband/hw/ipath/ipath_intr.c b/drivers/infiniband/hw/ipath/ipath_intr.c
index 3230edb..b3445e9 100644
--- a/drivers/infiniband/hw/ipath/ipath_intr.c
+++ b/drivers/infiniband/hw/ipath/ipath_intr.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -38,42 +38,12 @@
#include "ipath_verbs.h"
#include "ipath_common.h"
-/*
- * clear (write) a pio buffer, to clear a parity error. This routine
- * should only be called when in freeze mode, and the buffer should be
- * canceled afterwards.
- */
-static void ipath_clrpiobuf(struct ipath_devdata *dd, u32 pnum)
-{
- u32 __iomem *pbuf;
- u32 dwcnt; /* dword count to write */
- if (pnum < dd->ipath_piobcnt2k) {
- pbuf = (u32 __iomem *) (dd->ipath_pio2kbase + pnum *
- dd->ipath_palign);
- dwcnt = dd->ipath_piosize2k >> 2;
- }
- else {
- pbuf = (u32 __iomem *) (dd->ipath_pio4kbase +
- (pnum - dd->ipath_piobcnt2k) * dd->ipath_4kalign);
- dwcnt = dd->ipath_piosize4k >> 2;
- }
- dev_info(&dd->pcidev->dev,
- "Rewrite PIO buffer %u, to recover from parity error\n",
- pnum);
-
- /* no flush required, since already in freeze */
- writel(dwcnt + 1, pbuf);
- while (--dwcnt)
- writel(0, pbuf++);
-}
/*
* Called when we might have an error that is specific to a particular
* PIO buffer, and may need to cancel that buffer, so it can be re-used.
- * If rewrite is true, and bits are set in the sendbufferror registers,
- * we'll write to the buffer, for error recovery on parity errors.
*/
-void ipath_disarm_senderrbufs(struct ipath_devdata *dd, int rewrite)
+void ipath_disarm_senderrbufs(struct ipath_devdata *dd)
{
u32 piobcnt;
unsigned long sbuf[4];
@@ -109,11 +79,8 @@ void ipath_disarm_senderrbufs(struct ipath_devdata *dd, int rewrite)
}
for (i = 0; i < piobcnt; i++)
- if (test_bit(i, sbuf)) {
- if (rewrite)
- ipath_clrpiobuf(dd, i);
+ if (test_bit(i, sbuf))
ipath_disarm_piobufs(dd, i, 1);
- }
/* ignore armlaunch errs for a bit */
dd->ipath_lastcancel = jiffies+3;
}
@@ -164,7 +131,7 @@ static u64 handle_e_sum_errs(struct ipath_devdata *dd, ipath_err_t errs)
{
u64 ignore_this_time = 0;
- ipath_disarm_senderrbufs(dd, 0);
+ ipath_disarm_senderrbufs(dd);
if ((errs & E_SUM_LINK_PKTERRS) &&
!(dd->ipath_flags & IPATH_LINKACTIVE)) {
/*
@@ -302,6 +269,17 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
lastlstate = ipath_ib_linkstate(dd, dd->ipath_lastibcstat);
ltstate = ipath_ib_linktrstate(dd, ibcs); /* linktrainingtate */
+ /* Since going into a recovery state causes the link state to go
+ * down and since recovery is transitory, it is better if we "miss"
+ * ever seeing the link training state go into recovery (i.e.,
+ * ignore this transition for link state special handling purposes)
+ * without even updating ipath_lastibcstat.
+ */
+ if ((ltstate == INFINIPATH_IBCS_LT_STATE_RECOVERRETRAIN) ||
+ (ltstate == INFINIPATH_IBCS_LT_STATE_RECOVERWAITRMT) ||
+ (ltstate == INFINIPATH_IBCS_LT_STATE_RECOVERIDLE))
+ goto done;
+
/*
* if linkstate transitions into INIT from any of the various down
* states, or if it transitions from any of the up (INIT or better)
@@ -312,12 +290,14 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
lastlstate == INFINIPATH_IBCS_L_STATE_DOWN) {
/* transitioned to UP */
if (dd->ipath_f_ib_updown(dd, 1, ibcs)) {
+ /* link came up, so we must no longer be disabled */
+ dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED;
ipath_cdbg(LINKVERB, "LinkUp handled, skipped\n");
goto skip_ibchange; /* chip-code handled */
}
} else if ((lastlstate >= INFINIPATH_IBCS_L_STATE_INIT ||
(dd->ipath_flags & IPATH_IB_FORCE_NOTIFY)) &&
- ltstate <= INFINIPATH_IBCS_LT_STATE_CFGDEBOUNCE &&
+ ltstate <= INFINIPATH_IBCS_LT_STATE_CFGWAITRMT &&
ltstate != INFINIPATH_IBCS_LT_STATE_LINKUP) {
int handled;
handled = dd->ipath_f_ib_updown(dd, 0, ibcs);
@@ -354,7 +334,8 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
*/
if (lastlts == INFINIPATH_IBCS_LT_STATE_POLLACTIVE
|| lastlts == INFINIPATH_IBCS_LT_STATE_POLLQUIET) {
- if (++dd->ipath_ibpollcnt == 40) {
+ if (!(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG) &&
+ (++dd->ipath_ibpollcnt == 40)) {
dd->ipath_flags |= IPATH_NOCABLE;
*dd->ipath_statusp |=
IPATH_STATUS_IB_NOCABLE;
@@ -420,6 +401,8 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
dd->ipath_flags &= ~(IPATH_LINKUNK | IPATH_LINKINIT
| IPATH_LINKDOWN | IPATH_LINKARMED |
IPATH_NOCABLE);
+ if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
+ ipath_restart_sdma(dd);
signal_ib_event(dd, IB_EVENT_PORT_ACTIVE);
/* LED active not handled in chip _f_updown */
dd->ipath_f_setextled(dd, lstate, ltstate);
@@ -462,6 +445,8 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
skip_ibchange:
dd->ipath_lastibcstat = ibcs;
+done:
+ return;
}
@@ -474,12 +459,19 @@ static void handle_supp_msgs(struct ipath_devdata *dd,
*/
if (dd->ipath_lasterror & ~INFINIPATH_E_IBSTATUSCHANGED) {
int iserr;
- iserr = ipath_decode_err(msg, msgsz,
+ ipath_err_t mask;
+ iserr = ipath_decode_err(dd, msg, msgsz,
dd->ipath_lasterror &
~INFINIPATH_E_IBSTATUSCHANGED);
- if (dd->ipath_lasterror &
- ~(INFINIPATH_E_RRCVEGRFULL |
- INFINIPATH_E_RRCVHDRFULL | INFINIPATH_E_PKTERRS))
+
+ mask = INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL |
+ INFINIPATH_E_PKTERRS | INFINIPATH_E_SDMADISABLED;
+
+ /* if we're in debug, then don't mask SDMADISABLED msgs */
+ if (ipath_debug & __IPATH_DBG)
+ mask &= ~INFINIPATH_E_SDMADISABLED;
+
+ if (dd->ipath_lasterror & ~mask)
ipath_dev_err(dd, "Suppressed %u messages for "
"fast-repeating errors (%s) (%llx)\n",
supp_msgs, msg,
@@ -543,7 +535,7 @@ static void handle_sdma_errors(struct ipath_devdata *dd, ipath_err_t errs)
if (ipath_debug & __IPATH_DBG) {
char msg[128];
- ipath_decode_err(msg, sizeof msg, errs &
+ ipath_decode_err(dd, msg, sizeof msg, errs &
INFINIPATH_E_SDMAERRS);
ipath_dbg("errors %lx (%s)\n", (unsigned long)errs, msg);
}
@@ -706,7 +698,7 @@ static int handle_errors(struct ipath_devdata *dd, ipath_err_t errs)
dd->ipath_errormask &= ~dd->ipath_maskederrs;
ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
dd->ipath_errormask);
- s_iserr = ipath_decode_err(msg, sizeof msg,
+ s_iserr = ipath_decode_err(dd, msg, sizeof msg,
dd->ipath_maskederrs);
if (dd->ipath_maskederrs &
@@ -776,16 +768,23 @@ static int handle_errors(struct ipath_devdata *dd, ipath_err_t errs)
if (!errs)
return 0;
- if (!noprint)
+ if (!noprint) {
+ ipath_err_t mask;
/*
- * the ones we mask off are handled specially below or above
+ * The ones we mask off are handled specially below
+ * or above. Also mask SDMADISABLED by default as it
+ * is too chatty.
*/
- ipath_decode_err(msg, sizeof msg,
- errs & ~(INFINIPATH_E_IBSTATUSCHANGED |
- INFINIPATH_E_RRCVEGRFULL |
- INFINIPATH_E_RRCVHDRFULL |
- INFINIPATH_E_HARDWARE));
- else
+ mask = INFINIPATH_E_IBSTATUSCHANGED |
+ INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL |
+ INFINIPATH_E_HARDWARE | INFINIPATH_E_SDMADISABLED;
+
+ /* if we're in debug, then don't mask SDMADISABLED msgs */
+ if (ipath_debug & __IPATH_DBG)
+ mask &= ~INFINIPATH_E_SDMADISABLED;
+
+ ipath_decode_err(dd, msg, sizeof msg, errs & ~mask);
+ } else
/* so we don't need if (!noprint) at strlcat's below */
*msg = 0;
@@ -879,8 +878,8 @@ static int handle_errors(struct ipath_devdata *dd, ipath_err_t errs)
* processes (causing armlaunch), send errors due to going into freeze mode,
* etc., and try to avoid causing extra interrupts while doing so.
* Forcibly update the in-memory pioavail register copies after cleanup
- * because the chip won't do it for anything changing while in freeze mode
- * (we don't want to wait for the next pio buffer state change).
+ * because the chip won't do it while in freeze mode (the register values
+ * themselves are kept correct).
* Make sure that we don't lose any important interrupts by using the chip
* feature that says that writing 0 to a bit in *clear that is set in
* *status will cause an interrupt to be generated again (if allowed by
@@ -888,44 +887,23 @@ static int handle_errors(struct ipath_devdata *dd, ipath_err_t errs)
*/
void ipath_clear_freeze(struct ipath_devdata *dd)
{
- int i, im;
- __le64 val;
-
/* disable error interrupts, to avoid confusion */
ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask, 0ULL);
/* also disable interrupts; errormask is sometimes overwriten */
ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
- /*
- * clear all sends, because they have may been
- * completed by usercode while in freeze mode, and
- * therefore would not be sent, and eventually
- * might cause the process to run out of bufs
- */
- ipath_cancel_sends(dd, 0);
+ ipath_cancel_sends(dd, 1);
+
+ /* clear the freeze, and be sure chip saw it */
ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
dd->ipath_control);
+ ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
- /* ensure pio avail updates continue */
+ /* force in-memory update now we are out of freeze */
ipath_force_pio_avail_update(dd);
/*
- * We just enabled pioavailupdate, so dma copy is almost certainly
- * not yet right, so read the registers directly. Similar to init
- */
- for (i = 0; i < dd->ipath_pioavregs; i++) {
- /* deal with 6110 chip bug */
- im = (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) ?
- i ^ 1 : i;
- val = ipath_read_kreg64(dd, (0x1000 / sizeof(u64)) + im);
- dd->ipath_pioavailregs_dma[i] = dd->ipath_pioavailshadow[i]
- = le64_to_cpu(val) |
- (~dd->ipath_pioavailkernel[i] <<
- INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT);
- }
-
- /*
* force new interrupt if any hwerr, error or interrupt bits are
* still set, and clear "safe" send packet errors related to freeze
* and cancelling sends. Re-enable error interrupts before possible
@@ -1286,10 +1264,8 @@ irqreturn_t ipath_intr(int irq, void *data)
ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
- if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA))
- handle_layer_pioavail(dd);
- else
- ipath_dbg("unexpected BUFAVAIL intr\n");
+ /* always process; sdma verbs uses PIO for acks and VL15 */
+ handle_layer_pioavail(dd);
}
ret = IRQ_HANDLED;
diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h
index 5056e95..7754f5e 100644
--- a/drivers/infiniband/hw/ipath/ipath_kernel.h
+++ b/drivers/infiniband/hw/ipath/ipath_kernel.h
@@ -1,7 +1,7 @@
#ifndef _IPATH_KERNEL_H
#define _IPATH_KERNEL_H
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -118,6 +118,10 @@ struct ipath_portdata {
u16 port_subport_cnt;
/* non-zero if port is being shared. */
u16 port_subport_id;
+ /* number of pio bufs for this port (all procs, if shared) */
+ u32 port_piocnt;
+ /* first pio buffer for this port */
+ u32 port_pio_base;
/* chip offset of PIO buffers for this port */
u32 port_piobufs;
/* how many alloc_pages() chunks in port_rcvegrbuf_pages */
@@ -385,6 +389,8 @@ struct ipath_devdata {
u32 ipath_lastrpkts;
/* pio bufs allocated per port */
u32 ipath_pbufsport;
+ /* if remainder on bufs/port, ports < extrabuf get 1 extra */
+ u32 ipath_ports_extrabuf;
u32 ipath_pioupd_thresh; /* update threshold, some chips */
/*
* number of ports configured as max; zero is set to number chip
@@ -423,6 +429,13 @@ struct ipath_devdata {
u32 ipath_pcibar1;
u32 ipath_x1_fix_tries;
u32 ipath_autoneg_tries;
+ u32 serdes_first_init_done;
+
+ struct ipath_relock {
+ atomic_t ipath_relock_timer_active;
+ struct timer_list ipath_relock_timer;
+ unsigned int ipath_relock_interval; /* in jiffies */
+ } ipath_relock_singleton;
/* interrupt number */
int ipath_irq;
@@ -504,6 +517,7 @@ struct ipath_devdata {
/* HoL blocking / user app forward-progress state */
unsigned ipath_hol_state;
+ unsigned ipath_hol_next;
struct timer_list ipath_hol_timer;
/*
@@ -801,10 +815,12 @@ struct ipath_devdata {
u8 ipath_presets_needed; /* Set if presets to be restored next DOWN */
};
-/* HoL blocking / forward progress states */
+/* ipath_hol_state values (stopping/starting user proc, send flushing */
#define IPATH_HOL_UP 0
-#define IPATH_HOL_DOWNSTOP 1
-#define IPATH_HOL_DOWNCONT 2
+#define IPATH_HOL_DOWN 1
+/* ipath_hol_next toggle values, used when hol_state IPATH_HOL_DOWN */
+#define IPATH_HOL_DOWNSTOP 0
+#define IPATH_HOL_DOWNCONT 1
/* bit positions for sdma_status */
#define IPATH_SDMA_ABORTING 0
@@ -867,7 +883,8 @@ struct sk_buff *ipath_alloc_skb(struct ipath_devdata *dd, gfp_t);
extern int ipath_diag_inuse;
irqreturn_t ipath_intr(int irq, void *devid);
-int ipath_decode_err(char *buf, size_t blen, ipath_err_t err);
+int ipath_decode_err(struct ipath_devdata *dd, char *buf, size_t blen,
+ ipath_err_t err);
#if __IPATH_INFO || __IPATH_DBG
extern const char *ipath_ibcstatus_str[];
#endif
@@ -903,6 +920,10 @@ void ipath_disable_armlaunch(struct ipath_devdata *);
void ipath_hol_down(struct ipath_devdata *);
void ipath_hol_up(struct ipath_devdata *);
void ipath_hol_event(unsigned long);
+void ipath_toggle_rclkrls(struct ipath_devdata *);
+void ipath_sd7220_clr_ibpar(struct ipath_devdata *);
+void ipath_set_relock_poll(struct ipath_devdata *, int);
+void ipath_shutdown_relock_poll(struct ipath_devdata *);
/* for use in system calls, where we want to know device type, etc. */
#define port_fp(fp) ((struct ipath_filedata *)(fp)->private_data)->pd
@@ -973,6 +994,8 @@ void ipath_hol_event(unsigned long);
#define IPATH_HAS_MULT_IB_SPEED 0x8000000
#define IPATH_IB_AUTONEG_INPROG 0x10000000
#define IPATH_IB_AUTONEG_FAILED 0x20000000
+ /* Linkdown-disable intentionally, Do not attempt to bring up */
+#define IPATH_IB_LINK_DISABLED 0x40000000
#define IPATH_IB_FORCE_NOTIFY 0x80000000 /* force notify on next ib change */
/* Bits in GPIO for the added interrupts */
@@ -1003,7 +1026,7 @@ int ipath_update_eeprom_log(struct ipath_devdata *dd);
void ipath_inc_eeprom_err(struct ipath_devdata *dd, u32 eidx, u32 incr);
u64 ipath_snap_cntr(struct ipath_devdata *, ipath_creg);
void ipath_dump_lookup_output_queue(struct ipath_devdata *);
-void ipath_disarm_senderrbufs(struct ipath_devdata *, int);
+void ipath_disarm_senderrbufs(struct ipath_devdata *);
void ipath_force_pio_avail_update(struct ipath_devdata *);
void signal_ib_event(struct ipath_devdata *dd, enum ib_event_type ev);
@@ -1019,6 +1042,7 @@ void ipath_set_led_override(struct ipath_devdata *dd, unsigned int val);
/* send dma routines */
int setup_sdma(struct ipath_devdata *);
void teardown_sdma(struct ipath_devdata *);
+void ipath_restart_sdma(struct ipath_devdata *);
void ipath_sdma_intr(struct ipath_devdata *);
int ipath_sdma_verbs_send(struct ipath_devdata *, struct ipath_sge_state *,
u32, struct ipath_verbs_txreq *);
diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c
index 4f55b91..361a73f 100644
--- a/drivers/infiniband/hw/ipath/ipath_mad.c
+++ b/drivers/infiniband/hw/ipath/ipath_mad.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -591,6 +591,10 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
else
goto err;
ipath_set_linkstate(dd, lstate);
+ if (lstate == IPATH_IB_LINKDOWN_DISABLE) {
+ ret = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
+ goto done;
+ }
ipath_wait_linkstate(dd, IPATH_LINKINIT | IPATH_LINKARMED |
IPATH_LINKACTIVE, 1000);
break;
diff --git a/drivers/infiniband/hw/ipath/ipath_qp.c b/drivers/infiniband/hw/ipath/ipath_qp.c
index f43b9d4..e8498bb 100644
--- a/drivers/infiniband/hw/ipath/ipath_qp.c
+++ b/drivers/infiniband/hw/ipath/ipath_qp.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -242,7 +242,6 @@ static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp)
{
struct ipath_qp *q, **qpp;
unsigned long flags;
- int fnd = 0;
spin_lock_irqsave(&qpt->lock, flags);
@@ -253,51 +252,40 @@ static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp)
*qpp = qp->next;
qp->next = NULL;
atomic_dec(&qp->refcount);
- fnd = 1;
break;
}
}
spin_unlock_irqrestore(&qpt->lock, flags);
-
- if (!fnd)
- return;
-
- free_qpn(qpt, qp->ibqp.qp_num);
-
- wait_event(qp->wait, !atomic_read(&qp->refcount));
}
/**
- * ipath_free_all_qps - remove all QPs from the table
+ * ipath_free_all_qps - check for QPs still in use
* @qpt: the QP table to empty
+ *
+ * There should not be any QPs still in use.
+ * Free memory for table.
*/
-void ipath_free_all_qps(struct ipath_qp_table *qpt)
+unsigned ipath_free_all_qps(struct ipath_qp_table *qpt)
{
unsigned long flags;
- struct ipath_qp *qp, *nqp;
- u32 n;
+ struct ipath_qp *qp;
+ u32 n, qp_inuse = 0;
+ spin_lock_irqsave(&qpt->lock, flags);
for (n = 0; n < qpt->max; n++) {
- spin_lock_irqsave(&qpt->lock, flags);
qp = qpt->table[n];
qpt->table[n] = NULL;
- spin_unlock_irqrestore(&qpt->lock, flags);
-
- while (qp) {
- nqp = qp->next;
- free_qpn(qpt, qp->ibqp.qp_num);
- if (!atomic_dec_and_test(&qp->refcount) ||
- !ipath_destroy_qp(&qp->ibqp))
- ipath_dbg("QP memory leak!\n");
- qp = nqp;
- }
+
+ for (; qp; qp = qp->next)
+ qp_inuse++;
}
+ spin_unlock_irqrestore(&qpt->lock, flags);
- for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {
+ for (n = 0; n < ARRAY_SIZE(qpt->map); n++)
if (qpt->map[n].page)
- free_page((unsigned long)qpt->map[n].page);
- }
+ free_page((unsigned long) qpt->map[n].page);
+ return qp_inuse;
}
/**
@@ -336,11 +324,12 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
qp->remote_qpn = 0;
qp->qkey = 0;
qp->qp_access_flags = 0;
- qp->s_busy = 0;
+ atomic_set(&qp->s_dma_busy, 0);
qp->s_flags &= IPATH_S_SIGNAL_REQ_WR;
qp->s_hdrwords = 0;
qp->s_wqe = NULL;
qp->s_pkt_delay = 0;
+ qp->s_draining = 0;
qp->s_psn = 0;
qp->r_psn = 0;
qp->r_msn = 0;
@@ -353,7 +342,8 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
}
qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
qp->r_nak_state = 0;
- qp->r_wrid_valid = 0;
+ qp->r_aflags = 0;
+ qp->r_flags = 0;
qp->s_rnr_timeout = 0;
qp->s_head = 0;
qp->s_tail = 0;
@@ -361,7 +351,6 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
qp->s_last = 0;
qp->s_ssn = 1;
qp->s_lsn = 0;
- qp->s_wait_credit = 0;
memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue));
qp->r_head_ack_queue = 0;
qp->s_tail_ack_queue = 0;
@@ -370,17 +359,17 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
qp->r_rq.wq->head = 0;
qp->r_rq.wq->tail = 0;
}
- qp->r_reuse_sge = 0;
}
/**
- * ipath_error_qp - put a QP into an error state
- * @qp: the QP to put into an error state
+ * ipath_error_qp - put a QP into the error state
+ * @qp: the QP to put into the error state
* @err: the receive completion error to signal if a RWQE is active
*
* Flushes both send and receive work queues.
* Returns true if last WQE event should be generated.
* The QP s_lock should be held and interrupts disabled.
+ * If we are already in error state, just return.
*/
int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
@@ -389,8 +378,10 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
struct ib_wc wc;
int ret = 0;
- ipath_dbg("QP%d/%d in error state (%d)\n",
- qp->ibqp.qp_num, qp->remote_qpn, err);
+ if (qp->state == IB_QPS_ERR)
+ goto bail;
+
+ qp->state = IB_QPS_ERR;
spin_lock(&dev->pending_lock);
if (!list_empty(&qp->timerwait))
@@ -399,39 +390,21 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
list_del_init(&qp->piowait);
spin_unlock(&dev->pending_lock);
- wc.vendor_err = 0;
- wc.byte_len = 0;
- wc.imm_data = 0;
+ /* Schedule the sending tasklet to drain the send work queue. */
+ if (qp->s_last != qp->s_head)
+ ipath_schedule_send(qp);
+
+ memset(&wc, 0, sizeof(wc));
wc.qp = &qp->ibqp;
- wc.src_qp = 0;
- wc.wc_flags = 0;
- wc.pkey_index = 0;
- wc.slid = 0;
- wc.sl = 0;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
- if (qp->r_wrid_valid) {
- qp->r_wrid_valid = 0;
+ wc.opcode = IB_WC_RECV;
+
+ if (test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) {
wc.wr_id = qp->r_wr_id;
- wc.opcode = IB_WC_RECV;
wc.status = err;
ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
}
wc.status = IB_WC_WR_FLUSH_ERR;
- while (qp->s_last != qp->s_head) {
- struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
-
- wc.wr_id = wqe->wr.wr_id;
- wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- if (++qp->s_last >= qp->s_size)
- qp->s_last = 0;
- ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
- }
- qp->s_cur = qp->s_tail = qp->s_head;
- qp->s_hdrwords = 0;
- qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
-
if (qp->r_rq.wq) {
struct ipath_rwq *wq;
u32 head;
@@ -447,7 +420,6 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
tail = wq->tail;
if (tail >= qp->r_rq.size)
tail = 0;
- wc.opcode = IB_WC_RECV;
while (tail != head) {
wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;
if (++tail >= qp->r_rq.size)
@@ -460,6 +432,7 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
} else if (qp->ibqp.event_handler)
ret = 1;
+bail:
return ret;
}
@@ -478,11 +451,10 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
struct ipath_ibdev *dev = to_idev(ibqp->device);
struct ipath_qp *qp = to_iqp(ibqp);
enum ib_qp_state cur_state, new_state;
- unsigned long flags;
int lastwqe = 0;
int ret;
- spin_lock_irqsave(&qp->s_lock, flags);
+ spin_lock_irq(&qp->s_lock);
cur_state = attr_mask & IB_QP_CUR_STATE ?
attr->cur_qp_state : qp->state;
@@ -535,16 +507,42 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
switch (new_state) {
case IB_QPS_RESET:
+ if (qp->state != IB_QPS_RESET) {
+ qp->state = IB_QPS_RESET;
+ spin_lock(&dev->pending_lock);
+ if (!list_empty(&qp->timerwait))
+ list_del_init(&qp->timerwait);
+ if (!list_empty(&qp->piowait))
+ list_del_init(&qp->piowait);
+ spin_unlock(&dev->pending_lock);
+ qp->s_flags &= ~IPATH_S_ANY_WAIT;
+ spin_unlock_irq(&qp->s_lock);
+ /* Stop the sending tasklet */
+ tasklet_kill(&qp->s_task);
+ wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));
+ spin_lock_irq(&qp->s_lock);
+ }
ipath_reset_qp(qp, ibqp->qp_type);
break;
+ case IB_QPS_SQD:
+ qp->s_draining = qp->s_last != qp->s_cur;
+ qp->state = new_state;
+ break;
+
+ case IB_QPS_SQE:
+ if (qp->ibqp.qp_type == IB_QPT_RC)
+ goto inval;
+ qp->state = new_state;
+ break;
+
case IB_QPS_ERR:
lastwqe = ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);
break;
default:
+ qp->state = new_state;
break;
-
}
if (attr_mask & IB_QP_PKEY_INDEX)
@@ -597,8 +595,7 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
qp->s_max_rd_atomic = attr->max_rd_atomic;
- qp->state = new_state;
- spin_unlock_irqrestore(&qp->s_lock, flags);
+ spin_unlock_irq(&qp->s_lock);
if (lastwqe) {
struct ib_event ev;
@@ -612,7 +609,7 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
goto bail;
inval:
- spin_unlock_irqrestore(&qp->s_lock, flags);
+ spin_unlock_irq(&qp->s_lock);
ret = -EINVAL;
bail:
@@ -643,7 +640,7 @@ int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
attr->pkey_index = qp->s_pkey_index;
attr->alt_pkey_index = 0;
attr->en_sqd_async_notify = 0;
- attr->sq_draining = 0;
+ attr->sq_draining = qp->s_draining;
attr->max_rd_atomic = qp->s_max_rd_atomic;
attr->max_dest_rd_atomic = qp->r_max_rd_atomic;
attr->min_rnr_timer = qp->r_min_rnr_timer;
@@ -830,6 +827,7 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
spin_lock_init(&qp->r_rq.lock);
atomic_set(&qp->refcount, 0);
init_waitqueue_head(&qp->wait);
+ init_waitqueue_head(&qp->wait_dma);
tasklet_init(&qp->s_task, ipath_do_send, (unsigned long)qp);
INIT_LIST_HEAD(&qp->piowait);
INIT_LIST_HEAD(&qp->timerwait);
@@ -923,6 +921,7 @@ bail_ip:
else
vfree(qp->r_rq.wq);
ipath_free_qp(&dev->qp_table, qp);
+ free_qpn(&dev->qp_table, qp->ibqp.qp_num);
bail_qp:
kfree(qp);
bail_swq:
@@ -944,42 +943,44 @@ int ipath_destroy_qp(struct ib_qp *ibqp)
{
struct ipath_qp *qp = to_iqp(ibqp);
struct ipath_ibdev *dev = to_idev(ibqp->device);
- unsigned long flags;
- spin_lock_irqsave(&qp->s_lock, flags);
- qp->state = IB_QPS_ERR;
- spin_unlock_irqrestore(&qp->s_lock, flags);
- spin_lock(&dev->n_qps_lock);
- dev->n_qps_allocated--;
- spin_unlock(&dev->n_qps_lock);
- set_bit(IPATH_S_DESTROYING, &qp->s_busy);
+ /* Make sure HW and driver activity is stopped. */
+ spin_lock_irq(&qp->s_lock);
+ if (qp->state != IB_QPS_RESET) {
+ qp->state = IB_QPS_RESET;
+ spin_lock(&dev->pending_lock);
+ if (!list_empty(&qp->timerwait))
+ list_del_init(&qp->timerwait);
+ if (!list_empty(&qp->piowait))
+ list_del_init(&qp->piowait);
+ spin_unlock(&dev->pending_lock);
+ qp->s_flags &= ~IPATH_S_ANY_WAIT;
+ spin_unlock_irq(&qp->s_lock);
+ /* Stop the sending tasklet */
+ tasklet_kill(&qp->s_task);
+ wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));
+ } else
+ spin_unlock_irq(&qp->s_lock);
- /* Stop the sending tasklet. */
- tasklet_kill(&qp->s_task);
+ ipath_free_qp(&dev->qp_table, qp);
if (qp->s_tx) {
atomic_dec(&qp->refcount);
if (qp->s_tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEBUF)
kfree(qp->s_tx->txreq.map_addr);
+ spin_lock_irq(&dev->pending_lock);
+ list_add(&qp->s_tx->txreq.list, &dev->txreq_free);
+ spin_unlock_irq(&dev->pending_lock);
+ qp->s_tx = NULL;
}
- /* Make sure the QP isn't on the timeout list. */
- spin_lock_irqsave(&dev->pending_lock, flags);
- if (!list_empty(&qp->timerwait))
- list_del_init(&qp->timerwait);
- if (!list_empty(&qp->piowait))
- list_del_init(&qp->piowait);
- if (qp->s_tx)
- list_add(&qp->s_tx->txreq.list, &dev->txreq_free);
- spin_unlock_irqrestore(&dev->pending_lock, flags);
+ wait_event(qp->wait, !atomic_read(&qp->refcount));
- /*
- * Make sure that the QP is not in the QPN table so receive
- * interrupts will discard packets for this QP. XXX Also remove QP
- * from multicast table.
- */
- if (atomic_read(&qp->refcount) != 0)
- ipath_free_qp(&dev->qp_table, qp);
+ /* all user's cleaned up, mark it available */
+ free_qpn(&dev->qp_table, qp->ibqp.qp_num);
+ spin_lock(&dev->n_qps_lock);
+ dev->n_qps_allocated--;
+ spin_unlock(&dev->n_qps_lock);
if (qp->ip)
kref_put(&qp->ip->ref, ipath_release_mmap_info);
@@ -1024,48 +1025,6 @@ bail:
}
/**
- * ipath_sqerror_qp - put a QP's send queue into an error state
- * @qp: QP who's send queue will be put into an error state
- * @wc: the WC responsible for putting the QP in this state
- *
- * Flushes the send work queue.
- * The QP s_lock should be held and interrupts disabled.
- */
-
-void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc)
-{
- struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
- struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
-
- ipath_dbg("Send queue error on QP%d/%d: err: %d\n",
- qp->ibqp.qp_num, qp->remote_qpn, wc->status);
-
- spin_lock(&dev->pending_lock);
- if (!list_empty(&qp->timerwait))
- list_del_init(&qp->timerwait);
- if (!list_empty(&qp->piowait))
- list_del_init(&qp->piowait);
- spin_unlock(&dev->pending_lock);
-
- ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
- if (++qp->s_last >= qp->s_size)
- qp->s_last = 0;
-
- wc->status = IB_WC_WR_FLUSH_ERR;
-
- while (qp->s_last != qp->s_head) {
- wqe = get_swqe_ptr(qp, qp->s_last);
- wc->wr_id = wqe->wr.wr_id;
- wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
- if (++qp->s_last >= qp->s_size)
- qp->s_last = 0;
- }
- qp->s_cur = qp->s_tail = qp->s_head;
- qp->state = IB_QPS_SQE;
-}
-
-/**
* ipath_get_credit - flush the send work queue of a QP
* @qp: the qp who's send work queue to flush
* @aeth: the Acknowledge Extended Transport Header
@@ -1091,7 +1050,8 @@ void ipath_get_credit(struct ipath_qp *qp, u32 aeth)
}
/* Restart sending if it was blocked due to lack of credits. */
- if (qp->s_cur != qp->s_head &&
+ if ((qp->s_flags & IPATH_S_WAIT_SSN_CREDIT) &&
+ qp->s_cur != qp->s_head &&
(qp->s_lsn == (u32) -1 ||
ipath_cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn,
qp->s_lsn + 1) <= 0))
diff --git a/drivers/infiniband/hw/ipath/ipath_rc.c b/drivers/infiniband/hw/ipath/ipath_rc.c
index ac8eafd..9d1c0f8 100644
--- a/drivers/infiniband/hw/ipath/ipath_rc.c
+++ b/drivers/infiniband/hw/ipath/ipath_rc.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -48,6 +48,7 @@ static u32 restart_sge(struct ipath_sge_state *ss, struct ipath_swqe *wqe,
ss->sge = wqe->sg_list[0];
ss->sg_list = wqe->sg_list + 1;
ss->num_sge = wqe->wr.num_sge;
+ ss->total_len = wqe->length;
ipath_skip_sge(ss, len);
return wqe->length - len;
}
@@ -92,6 +93,10 @@ static int ipath_make_rc_ack(struct ipath_ibdev *dev, struct ipath_qp *qp,
u32 bth0;
u32 bth2;
+ /* Don't send an ACK if we aren't supposed to. */
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
+ goto bail;
+
/* header size in 32-bit words LRH+BTH = (8+12)/4. */
hwords = 5;
@@ -238,14 +243,25 @@ int ipath_make_rc_req(struct ipath_qp *qp)
ipath_make_rc_ack(dev, qp, ohdr, pmtu))
goto done;
- if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) ||
- qp->s_rnr_timeout || qp->s_wait_credit)
- goto bail;
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) {
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))
+ goto bail;
+ /* We are in the error state, flush the work request. */
+ if (qp->s_last == qp->s_head)
+ goto bail;
+ /* If DMAs are in progress, we can't flush immediately. */
+ if (atomic_read(&qp->s_dma_busy)) {
+ qp->s_flags |= IPATH_S_WAIT_DMA;
+ goto bail;
+ }
+ wqe = get_swqe_ptr(qp, qp->s_last);
+ ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
+ goto done;
+ }
- /* Limit the number of packets sent without an ACK. */
- if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT) > 0) {
- qp->s_wait_credit = 1;
- dev->n_rc_stalls++;
+ /* Leave BUSY set until RNR timeout. */
+ if (qp->s_rnr_timeout) {
+ qp->s_flags |= IPATH_S_WAITING;
goto bail;
}
@@ -257,6 +273,9 @@ int ipath_make_rc_req(struct ipath_qp *qp)
wqe = get_swqe_ptr(qp, qp->s_cur);
switch (qp->s_state) {
default:
+ if (!(ib_ipath_state_ops[qp->state] &
+ IPATH_PROCESS_NEXT_SEND_OK))
+ goto bail;
/*
* Resend an old request or start a new one.
*
@@ -294,8 +313,10 @@ int ipath_make_rc_req(struct ipath_qp *qp)
case IB_WR_SEND_WITH_IMM:
/* If no credit, return. */
if (qp->s_lsn != (u32) -1 &&
- ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0)
+ ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+ qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT;
goto bail;
+ }
wqe->lpsn = wqe->psn;
if (len > pmtu) {
wqe->lpsn += (len - 1) / pmtu;
@@ -325,8 +346,10 @@ int ipath_make_rc_req(struct ipath_qp *qp)
case IB_WR_RDMA_WRITE_WITH_IMM:
/* If no credit, return. */
if (qp->s_lsn != (u32) -1 &&
- ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0)
+ ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+ qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT;
goto bail;
+ }
ohdr->u.rc.reth.vaddr =
cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
ohdr->u.rc.reth.rkey =
@@ -439,6 +462,7 @@ int ipath_make_rc_req(struct ipath_qp *qp)
qp->s_sge.sge = wqe->sg_list[0];
qp->s_sge.sg_list = wqe->sg_list + 1;
qp->s_sge.num_sge = wqe->wr.num_sge;
+ qp->s_sge.total_len = wqe->length;
qp->s_len = wqe->length;
if (newreq) {
qp->s_tail++;
@@ -570,7 +594,11 @@ int ipath_make_rc_req(struct ipath_qp *qp)
ipath_make_ruc_header(dev, qp, ohdr, bth0 | (qp->s_state << 24), bth2);
done:
ret = 1;
+ goto unlock;
+
bail:
+ qp->s_flags &= ~IPATH_S_BUSY;
+unlock:
spin_unlock_irqrestore(&qp->s_lock, flags);
return ret;
}
@@ -607,7 +635,11 @@ static void send_rc_ack(struct ipath_qp *qp)
spin_unlock_irqrestore(&qp->s_lock, flags);
+ /* Don't try to send ACKs if the link isn't ACTIVE */
dd = dev->dd;
+ if (!(dd->ipath_flags & IPATH_LINKACTIVE))
+ goto done;
+
piobuf = ipath_getpiobuf(dd, 0, &pbufn);
if (!piobuf) {
/*
@@ -677,15 +709,16 @@ static void send_rc_ack(struct ipath_qp *qp)
goto done;
queue_ack:
- dev->n_rc_qacks++;
- qp->s_flags |= IPATH_S_ACK_PENDING;
- qp->s_nak_state = qp->r_nak_state;
- qp->s_ack_psn = qp->r_ack_psn;
- spin_unlock_irqrestore(&qp->s_lock, flags);
-
- /* Schedule the send tasklet. */
- ipath_schedule_send(qp);
+ if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK) {
+ dev->n_rc_qacks++;
+ qp->s_flags |= IPATH_S_ACK_PENDING;
+ qp->s_nak_state = qp->r_nak_state;
+ qp->s_ack_psn = qp->r_ack_psn;
+ /* Schedule the send tasklet. */
+ ipath_schedule_send(qp);
+ }
+ spin_unlock_irqrestore(&qp->s_lock, flags);
done:
return;
}
@@ -780,27 +813,14 @@ done:
*
* The QP s_lock should be held and interrupts disabled.
*/
-void ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc)
+void ipath_restart_rc(struct ipath_qp *qp, u32 psn)
{
struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
struct ipath_ibdev *dev;
if (qp->s_retry == 0) {
- wc->wr_id = wqe->wr.wr_id;
- wc->status = IB_WC_RETRY_EXC_ERR;
- wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- wc->vendor_err = 0;
- wc->byte_len = 0;
- wc->qp = &qp->ibqp;
- wc->imm_data = 0;
- wc->src_qp = qp->remote_qpn;
- wc->wc_flags = 0;
- wc->pkey_index = 0;
- wc->slid = qp->remote_ah_attr.dlid;
- wc->sl = qp->remote_ah_attr.sl;
- wc->dlid_path_bits = 0;
- wc->port_num = 0;
- ipath_sqerror_qp(qp, wc);
+ ipath_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
+ ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);
goto bail;
}
qp->s_retry--;
@@ -813,6 +833,8 @@ void ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc)
spin_lock(&dev->pending_lock);
if (!list_empty(&qp->timerwait))
list_del_init(&qp->timerwait);
+ if (!list_empty(&qp->piowait))
+ list_del_init(&qp->piowait);
spin_unlock(&dev->pending_lock);
if (wqe->wr.opcode == IB_WR_RDMA_READ)
@@ -829,13 +851,7 @@ bail:
static inline void update_last_psn(struct ipath_qp *qp, u32 psn)
{
- if (qp->s_last_psn != psn) {
- qp->s_last_psn = psn;
- if (qp->s_wait_credit) {
- qp->s_wait_credit = 0;
- ipath_schedule_send(qp);
- }
- }
+ qp->s_last_psn = psn;
}
/**
@@ -854,6 +870,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
{
struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
struct ib_wc wc;
+ enum ib_wc_status status;
struct ipath_swqe *wqe;
int ret = 0;
u32 ack_psn;
@@ -918,7 +935,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
*/
update_last_psn(qp, wqe->psn - 1);
/* Retry this request. */
- ipath_restart_rc(qp, wqe->psn, &wc);
+ ipath_restart_rc(qp, wqe->psn);
/*
* No need to process the ACK/NAK since we are
* restarting an earlier request.
@@ -934,32 +951,23 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
qp->s_num_rd_atomic--;
/* Restart sending task if fence is complete */
- if ((qp->s_flags & IPATH_S_FENCE_PENDING) &&
- !qp->s_num_rd_atomic) {
- qp->s_flags &= ~IPATH_S_FENCE_PENDING;
+ if (((qp->s_flags & IPATH_S_FENCE_PENDING) &&
+ !qp->s_num_rd_atomic) ||
+ qp->s_flags & IPATH_S_RDMAR_PENDING)
ipath_schedule_send(qp);
- } else if (qp->s_flags & IPATH_S_RDMAR_PENDING) {
- qp->s_flags &= ~IPATH_S_RDMAR_PENDING;
- ipath_schedule_send(qp);
- }
}
/* Post a send completion queue entry if requested. */
if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||
(wqe->wr.send_flags & IB_SEND_SIGNALED)) {
+ memset(&wc, 0, sizeof wc);
wc.wr_id = wqe->wr.wr_id;
wc.status = IB_WC_SUCCESS;
wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- wc.vendor_err = 0;
wc.byte_len = wqe->length;
- wc.imm_data = 0;
wc.qp = &qp->ibqp;
wc.src_qp = qp->remote_qpn;
- wc.wc_flags = 0;
- wc.pkey_index = 0;
wc.slid = qp->remote_ah_attr.dlid;
wc.sl = qp->remote_ah_attr.sl;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
}
qp->s_retry = qp->s_retry_cnt;
@@ -980,6 +988,8 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
} else {
if (++qp->s_last >= qp->s_size)
qp->s_last = 0;
+ if (qp->state == IB_QPS_SQD && qp->s_last == qp->s_cur)
+ qp->s_draining = 0;
if (qp->s_last == qp->s_tail)
break;
wqe = get_swqe_ptr(qp, qp->s_last);
@@ -1021,7 +1031,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
if (qp->s_last == qp->s_tail)
goto bail;
if (qp->s_rnr_retry == 0) {
- wc.status = IB_WC_RNR_RETRY_EXC_ERR;
+ status = IB_WC_RNR_RETRY_EXC_ERR;
goto class_b;
}
if (qp->s_rnr_retry_cnt < 7)
@@ -1042,6 +1052,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
ib_ipath_rnr_table[(aeth >> IPATH_AETH_CREDIT_SHIFT) &
IPATH_AETH_CREDIT_MASK];
ipath_insert_rnr_queue(qp);
+ ipath_schedule_send(qp);
goto bail;
case 3: /* NAK */
@@ -1059,37 +1070,25 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
* RDMA READ response which terminates the RDMA
* READ.
*/
- ipath_restart_rc(qp, psn, &wc);
+ ipath_restart_rc(qp, psn);
break;
case 1: /* Invalid Request */
- wc.status = IB_WC_REM_INV_REQ_ERR;
+ status = IB_WC_REM_INV_REQ_ERR;
dev->n_other_naks++;
goto class_b;
case 2: /* Remote Access Error */
- wc.status = IB_WC_REM_ACCESS_ERR;
+ status = IB_WC_REM_ACCESS_ERR;
dev->n_other_naks++;
goto class_b;
case 3: /* Remote Operation Error */
- wc.status = IB_WC_REM_OP_ERR;
+ status = IB_WC_REM_OP_ERR;
dev->n_other_naks++;
class_b:
- wc.wr_id = wqe->wr.wr_id;
- wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- wc.vendor_err = 0;
- wc.byte_len = 0;
- wc.qp = &qp->ibqp;
- wc.imm_data = 0;
- wc.src_qp = qp->remote_qpn;
- wc.wc_flags = 0;
- wc.pkey_index = 0;
- wc.slid = qp->remote_ah_attr.dlid;
- wc.sl = qp->remote_ah_attr.sl;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
- ipath_sqerror_qp(qp, &wc);
+ ipath_send_complete(qp, wqe, status);
+ ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);
break;
default:
@@ -1135,8 +1134,8 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
int header_in_data)
{
struct ipath_swqe *wqe;
+ enum ib_wc_status status;
unsigned long flags;
- struct ib_wc wc;
int diff;
u32 pad;
u32 aeth;
@@ -1144,6 +1143,10 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
spin_lock_irqsave(&qp->s_lock, flags);
+ /* Double check we can process this now that we hold the s_lock. */
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
+ goto ack_done;
+
/* Ignore invalid responses. */
if (ipath_cmp24(psn, qp->s_next_psn) >= 0)
goto ack_done;
@@ -1168,6 +1171,7 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
if (unlikely(qp->s_last == qp->s_tail))
goto ack_done;
wqe = get_swqe_ptr(qp, qp->s_last);
+ status = IB_WC_SUCCESS;
switch (opcode) {
case OP(ACKNOWLEDGE):
@@ -1196,6 +1200,7 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
wqe = get_swqe_ptr(qp, qp->s_last);
if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
goto ack_op_err;
+ qp->r_flags &= ~IPATH_R_RDMAR_SEQ;
/*
* If this is a response to a resent RDMA read, we
* have to be careful to copy the data to the right
@@ -1209,7 +1214,10 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
/* no AETH, no ACK */
if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) {
dev->n_rdma_seq++;
- ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
+ if (qp->r_flags & IPATH_R_RDMAR_SEQ)
+ goto ack_done;
+ qp->r_flags |= IPATH_R_RDMAR_SEQ;
+ ipath_restart_rc(qp, qp->s_last_psn + 1);
goto ack_done;
}
if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
@@ -1270,7 +1278,10 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
/* ACKs READ req. */
if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) {
dev->n_rdma_seq++;
- ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
+ if (qp->r_flags & IPATH_R_RDMAR_SEQ)
+ goto ack_done;
+ qp->r_flags |= IPATH_R_RDMAR_SEQ;
+ ipath_restart_rc(qp, qp->s_last_psn + 1);
goto ack_done;
}
if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
@@ -1300,31 +1311,16 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
goto ack_done;
}
-ack_done:
- spin_unlock_irqrestore(&qp->s_lock, flags);
- goto bail;
-
ack_op_err:
- wc.status = IB_WC_LOC_QP_OP_ERR;
+ status = IB_WC_LOC_QP_OP_ERR;
goto ack_err;
ack_len_err:
- wc.status = IB_WC_LOC_LEN_ERR;
+ status = IB_WC_LOC_LEN_ERR;
ack_err:
- wc.wr_id = wqe->wr.wr_id;
- wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- wc.vendor_err = 0;
- wc.byte_len = 0;
- wc.imm_data = 0;
- wc.qp = &qp->ibqp;
- wc.src_qp = qp->remote_qpn;
- wc.wc_flags = 0;
- wc.pkey_index = 0;
- wc.slid = qp->remote_ah_attr.dlid;
- wc.sl = qp->remote_ah_attr.sl;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
- ipath_sqerror_qp(qp, &wc);
+ ipath_send_complete(qp, wqe, status);
+ ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);
+ack_done:
spin_unlock_irqrestore(&qp->s_lock, flags);
bail:
return;
@@ -1393,7 +1389,12 @@ static inline int ipath_rc_rcv_error(struct ipath_ibdev *dev,
psn &= IPATH_PSN_MASK;
e = NULL;
old_req = 1;
+
spin_lock_irqsave(&qp->s_lock, flags);
+ /* Double check we can process this now that we hold the s_lock. */
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
+ goto unlock_done;
+
for (i = qp->r_head_ack_queue; ; i = prev) {
if (i == qp->s_tail_ack_queue)
old_req = 0;
@@ -1532,13 +1533,12 @@ send_ack:
return 0;
}
-static void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err)
+void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err)
{
unsigned long flags;
int lastwqe;
spin_lock_irqsave(&qp->s_lock, flags);
- qp->state = IB_QPS_ERR;
lastwqe = ipath_error_qp(qp, err);
spin_unlock_irqrestore(&qp->s_lock, flags);
@@ -1554,18 +1554,15 @@ static void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err)
static inline void ipath_update_ack_queue(struct ipath_qp *qp, unsigned n)
{
- unsigned long flags;
unsigned next;
next = n + 1;
if (next > IPATH_MAX_RDMA_ATOMIC)
next = 0;
- spin_lock_irqsave(&qp->s_lock, flags);
if (n == qp->s_tail_ack_queue) {
qp->s_tail_ack_queue = next;
qp->s_ack_state = OP(ACKNOWLEDGE);
}
- spin_unlock_irqrestore(&qp->s_lock, flags);
}
/**
@@ -1594,6 +1591,7 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
int diff;
struct ib_reth *reth;
int header_in_data;
+ unsigned long flags;
/* Validate the SLID. See Ch. 9.6.1.5 */
if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid))
@@ -1652,11 +1650,7 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
opcode == OP(SEND_LAST) ||
opcode == OP(SEND_LAST_WITH_IMMEDIATE))
break;
- nack_inv:
- ipath_rc_error(qp, IB_WC_REM_INV_REQ_ERR);
- qp->r_nak_state = IB_NAK_INVALID_REQUEST;
- qp->r_ack_psn = qp->r_psn;
- goto send_ack;
+ goto nack_inv;
case OP(RDMA_WRITE_FIRST):
case OP(RDMA_WRITE_MIDDLE):
@@ -1682,18 +1676,13 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
break;
}
- wc.imm_data = 0;
- wc.wc_flags = 0;
+ memset(&wc, 0, sizeof wc);
/* OK, process the packet. */
switch (opcode) {
case OP(SEND_FIRST):
- if (!ipath_get_rwqe(qp, 0)) {
- rnr_nak:
- qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer;
- qp->r_ack_psn = qp->r_psn;
- goto send_ack;
- }
+ if (!ipath_get_rwqe(qp, 0))
+ goto rnr_nak;
qp->r_rcv_len = 0;
/* FALLTHROUGH */
case OP(SEND_MIDDLE):
@@ -1750,20 +1739,19 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
goto nack_inv;
ipath_copy_sge(&qp->r_sge, data, tlen);
qp->r_msn++;
- if (!qp->r_wrid_valid)
+ if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))
break;
- qp->r_wrid_valid = 0;
wc.wr_id = qp->r_wr_id;
wc.status = IB_WC_SUCCESS;
- wc.opcode = IB_WC_RECV;
- wc.vendor_err = 0;
+ if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) ||
+ opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
+ wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
+ else
+ wc.opcode = IB_WC_RECV;
wc.qp = &qp->ibqp;
wc.src_qp = qp->remote_qpn;
- wc.pkey_index = 0;
wc.slid = qp->remote_ah_attr.dlid;
wc.sl = qp->remote_ah_attr.sl;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
/* Signal completion event if the solicited bit is set. */
ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
(ohdr->bth[0] &
@@ -1824,9 +1812,13 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
next = qp->r_head_ack_queue + 1;
if (next > IPATH_MAX_RDMA_ATOMIC)
next = 0;
+ spin_lock_irqsave(&qp->s_lock, flags);
+ /* Double check we can process this while holding the s_lock. */
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
+ goto unlock;
if (unlikely(next == qp->s_tail_ack_queue)) {
if (!qp->s_ack_queue[next].sent)
- goto nack_inv;
+ goto nack_inv_unlck;
ipath_update_ack_queue(qp, next);
}
e = &qp->s_ack_queue[qp->r_head_ack_queue];
@@ -1847,7 +1839,7 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr,
rkey, IB_ACCESS_REMOTE_READ);
if (unlikely(!ok))
- goto nack_acc;
+ goto nack_acc_unlck;
/*
* Update the next expected PSN. We add 1 later
* below, so only add the remainder here.
@@ -1874,13 +1866,12 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
qp->r_psn++;
qp->r_state = opcode;
qp->r_nak_state = 0;
- barrier();
qp->r_head_ack_queue = next;
/* Schedule the send tasklet. */
ipath_schedule_send(qp);
- goto done;
+ goto unlock;
}
case OP(COMPARE_SWAP):
@@ -1899,9 +1890,13 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
next = qp->r_head_ack_queue + 1;
if (next > IPATH_MAX_RDMA_ATOMIC)
next = 0;
+ spin_lock_irqsave(&qp->s_lock, flags);
+ /* Double check we can process this while holding the s_lock. */
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
+ goto unlock;
if (unlikely(next == qp->s_tail_ack_queue)) {
if (!qp->s_ack_queue[next].sent)
- goto nack_inv;
+ goto nack_inv_unlck;
ipath_update_ack_queue(qp, next);
}
if (!header_in_data)
@@ -1911,13 +1906,13 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) |
be32_to_cpu(ateth->vaddr[1]);
if (unlikely(vaddr & (sizeof(u64) - 1)))
- goto nack_inv;
+ goto nack_inv_unlck;
rkey = be32_to_cpu(ateth->rkey);
/* Check rkey & NAK */
if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge,
sizeof(u64), vaddr, rkey,
IB_ACCESS_REMOTE_ATOMIC)))
- goto nack_acc;
+ goto nack_acc_unlck;
/* Perform atomic OP and save result. */
maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
sdata = be64_to_cpu(ateth->swap_data);
@@ -1934,13 +1929,12 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
qp->r_psn++;
qp->r_state = opcode;
qp->r_nak_state = 0;
- barrier();
qp->r_head_ack_queue = next;
/* Schedule the send tasklet. */
ipath_schedule_send(qp);
- goto done;
+ goto unlock;
}
default:
@@ -1956,14 +1950,31 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
goto send_ack;
goto done;
+rnr_nak:
+ qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer;
+ qp->r_ack_psn = qp->r_psn;
+ goto send_ack;
+
+nack_inv_unlck:
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+nack_inv:
+ ipath_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
+ qp->r_nak_state = IB_NAK_INVALID_REQUEST;
+ qp->r_ack_psn = qp->r_psn;
+ goto send_ack;
+
+nack_acc_unlck:
+ spin_unlock_irqrestore(&qp->s_lock, flags);
nack_acc:
- ipath_rc_error(qp, IB_WC_REM_ACCESS_ERR);
+ ipath_rc_error(qp, IB_WC_LOC_PROT_ERR);
qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
qp->r_ack_psn = qp->r_psn;
-
send_ack:
send_rc_ack(qp);
+ goto done;
+unlock:
+ spin_unlock_irqrestore(&qp->s_lock, flags);
done:
return;
}
diff --git a/drivers/infiniband/hw/ipath/ipath_ruc.c b/drivers/infiniband/hw/ipath/ipath_ruc.c
index bcaa291..1cc5da6 100644
--- a/drivers/infiniband/hw/ipath/ipath_ruc.c
+++ b/drivers/infiniband/hw/ipath/ipath_ruc.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -78,6 +78,7 @@ const u32 ib_ipath_rnr_table[32] = {
* ipath_insert_rnr_queue - put QP on the RNR timeout list for the device
* @qp: the QP
*
+ * Called with the QP s_lock held and interrupts disabled.
* XXX Use a simple list for now. We might need a priority
* queue if we have lots of QPs waiting for RNR timeouts
* but that should be rare.
@@ -85,9 +86,9 @@ const u32 ib_ipath_rnr_table[32] = {
void ipath_insert_rnr_queue(struct ipath_qp *qp)
{
struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
- unsigned long flags;
- spin_lock_irqsave(&dev->pending_lock, flags);
+ /* We already did a spin_lock_irqsave(), so just use spin_lock */
+ spin_lock(&dev->pending_lock);
if (list_empty(&dev->rnrwait))
list_add(&qp->timerwait, &dev->rnrwait);
else {
@@ -109,7 +110,7 @@ void ipath_insert_rnr_queue(struct ipath_qp *qp)
nqp->s_rnr_timeout -= qp->s_rnr_timeout;
list_add(&qp->timerwait, l);
}
- spin_unlock_irqrestore(&dev->pending_lock, flags);
+ spin_unlock(&dev->pending_lock);
}
/**
@@ -136,24 +137,16 @@ int ipath_init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe,
j++;
}
ss->num_sge = j;
+ ss->total_len = *lengthp;
ret = 1;
goto bail;
bad_lkey:
+ memset(&wc, 0, sizeof(wc));
wc.wr_id = wqe->wr_id;
wc.status = IB_WC_LOC_PROT_ERR;
wc.opcode = IB_WC_RECV;
- wc.vendor_err = 0;
- wc.byte_len = 0;
- wc.imm_data = 0;
wc.qp = &qp->ibqp;
- wc.src_qp = 0;
- wc.wc_flags = 0;
- wc.pkey_index = 0;
- wc.slid = 0;
- wc.sl = 0;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
/* Signal solicited completion event. */
ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
ret = 0;
@@ -194,6 +187,11 @@ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
}
spin_lock_irqsave(&rq->lock, flags);
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
+ ret = 0;
+ goto unlock;
+ }
+
wq = rq->wq;
tail = wq->tail;
/* Validate tail before using it since it is user writable. */
@@ -201,9 +199,8 @@ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
tail = 0;
do {
if (unlikely(tail == wq->head)) {
- spin_unlock_irqrestore(&rq->lock, flags);
ret = 0;
- goto bail;
+ goto unlock;
}
/* Make sure entry is read after head index is read. */
smp_rmb();
@@ -216,7 +213,7 @@ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
wq->tail = tail;
ret = 1;
- qp->r_wrid_valid = 1;
+ set_bit(IPATH_R_WRID_VALID, &qp->r_aflags);
if (handler) {
u32 n;
@@ -243,8 +240,8 @@ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
goto bail;
}
}
+unlock:
spin_unlock_irqrestore(&rq->lock, flags);
-
bail:
return ret;
}
@@ -270,38 +267,63 @@ static void ipath_ruc_loopback(struct ipath_qp *sqp)
struct ib_wc wc;
u64 sdata;
atomic64_t *maddr;
+ enum ib_wc_status send_status;
+ /*
+ * Note that we check the responder QP state after
+ * checking the requester's state.
+ */
qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn);
- if (!qp) {
- dev->n_pkt_drops++;
- return;
- }
-again:
spin_lock_irqsave(&sqp->s_lock, flags);
- if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_SEND_OK) ||
- sqp->s_rnr_timeout) {
- spin_unlock_irqrestore(&sqp->s_lock, flags);
- goto done;
- }
+ /* Return if we are already busy processing a work request. */
+ if ((sqp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) ||
+ !(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_OR_FLUSH_SEND))
+ goto unlock;
- /* Get the next send request. */
- if (sqp->s_last == sqp->s_head) {
- /* Send work queue is empty. */
- spin_unlock_irqrestore(&sqp->s_lock, flags);
- goto done;
+ sqp->s_flags |= IPATH_S_BUSY;
+
+again:
+ if (sqp->s_last == sqp->s_head)
+ goto clr_busy;
+ wqe = get_swqe_ptr(sqp, sqp->s_last);
+
+ /* Return if it is not OK to start a new work reqeust. */
+ if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_NEXT_SEND_OK)) {
+ if (!(ib_ipath_state_ops[sqp->state] & IPATH_FLUSH_SEND))
+ goto clr_busy;
+ /* We are in the error state, flush the work request. */
+ send_status = IB_WC_WR_FLUSH_ERR;
+ goto flush_send;
}
/*
* We can rely on the entry not changing without the s_lock
* being held until we update s_last.
+ * We increment s_cur to indicate s_last is in progress.
*/
- wqe = get_swqe_ptr(sqp, sqp->s_last);
+ if (sqp->s_last == sqp->s_cur) {
+ if (++sqp->s_cur >= sqp->s_size)
+ sqp->s_cur = 0;
+ }
spin_unlock_irqrestore(&sqp->s_lock, flags);
- wc.wc_flags = 0;
- wc.imm_data = 0;
+ if (!qp || !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
+ dev->n_pkt_drops++;
+ /*
+ * For RC, the requester would timeout and retry so
+ * shortcut the timeouts and just signal too many retries.
+ */
+ if (sqp->ibqp.qp_type == IB_QPT_RC)
+ send_status = IB_WC_RETRY_EXC_ERR;
+ else
+ send_status = IB_WC_SUCCESS;
+ goto serr;
+ }
+
+ memset(&wc, 0, sizeof wc);
+ send_status = IB_WC_SUCCESS;
sqp->s_sge.sge = wqe->sg_list[0];
sqp->s_sge.sg_list = wqe->sg_list + 1;
@@ -313,75 +335,34 @@ again:
wc.imm_data = wqe->wr.imm_data;
/* FALLTHROUGH */
case IB_WR_SEND:
- if (!ipath_get_rwqe(qp, 0)) {
- rnr_nak:
- /* Handle RNR NAK */
- if (qp->ibqp.qp_type == IB_QPT_UC)
- goto send_comp;
- if (sqp->s_rnr_retry == 0) {
- wc.status = IB_WC_RNR_RETRY_EXC_ERR;
- goto err;
- }
- if (sqp->s_rnr_retry_cnt < 7)
- sqp->s_rnr_retry--;
- dev->n_rnr_naks++;
- sqp->s_rnr_timeout =
- ib_ipath_rnr_table[qp->r_min_rnr_timer];
- ipath_insert_rnr_queue(sqp);
- goto done;
- }
+ if (!ipath_get_rwqe(qp, 0))
+ goto rnr_nak;
break;
case IB_WR_RDMA_WRITE_WITH_IMM:
- if (unlikely(!(qp->qp_access_flags &
- IB_ACCESS_REMOTE_WRITE))) {
- wc.status = IB_WC_REM_INV_REQ_ERR;
- goto err;
- }
+ if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
+ goto inv_err;
wc.wc_flags = IB_WC_WITH_IMM;
wc.imm_data = wqe->wr.imm_data;
if (!ipath_get_rwqe(qp, 1))
goto rnr_nak;
/* FALLTHROUGH */
case IB_WR_RDMA_WRITE:
- if (unlikely(!(qp->qp_access_flags &
- IB_ACCESS_REMOTE_WRITE))) {
- wc.status = IB_WC_REM_INV_REQ_ERR;
- goto err;
- }
+ if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
+ goto inv_err;
if (wqe->length == 0)
break;
if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, wqe->length,
wqe->wr.wr.rdma.remote_addr,
wqe->wr.wr.rdma.rkey,
- IB_ACCESS_REMOTE_WRITE))) {
- acc_err:
- wc.status = IB_WC_REM_ACCESS_ERR;
- err:
- wc.wr_id = wqe->wr.wr_id;
- wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- wc.vendor_err = 0;
- wc.byte_len = 0;
- wc.qp = &sqp->ibqp;
- wc.src_qp = sqp->remote_qpn;
- wc.pkey_index = 0;
- wc.slid = sqp->remote_ah_attr.dlid;
- wc.sl = sqp->remote_ah_attr.sl;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
- spin_lock_irqsave(&sqp->s_lock, flags);
- ipath_sqerror_qp(sqp, &wc);
- spin_unlock_irqrestore(&sqp->s_lock, flags);
- goto done;
- }
+ IB_ACCESS_REMOTE_WRITE)))
+ goto acc_err;
+ qp->r_sge.total_len = wqe->length;
break;
case IB_WR_RDMA_READ:
- if (unlikely(!(qp->qp_access_flags &
- IB_ACCESS_REMOTE_READ))) {
- wc.status = IB_WC_REM_INV_REQ_ERR;
- goto err;
- }
+ if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
+ goto inv_err;
if (unlikely(!ipath_rkey_ok(qp, &sqp->s_sge, wqe->length,
wqe->wr.wr.rdma.remote_addr,
wqe->wr.wr.rdma.rkey,
@@ -390,15 +371,13 @@ again:
qp->r_sge.sge = wqe->sg_list[0];
qp->r_sge.sg_list = wqe->sg_list + 1;
qp->r_sge.num_sge = wqe->wr.num_sge;
+ qp->r_sge.total_len = wqe->length;
break;
case IB_WR_ATOMIC_CMP_AND_SWP:
case IB_WR_ATOMIC_FETCH_AND_ADD:
- if (unlikely(!(qp->qp_access_flags &
- IB_ACCESS_REMOTE_ATOMIC))) {
- wc.status = IB_WC_REM_INV_REQ_ERR;
- goto err;
- }
+ if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
+ goto inv_err;
if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, sizeof(u64),
wqe->wr.wr.atomic.remote_addr,
wqe->wr.wr.atomic.rkey,
@@ -415,7 +394,8 @@ again:
goto send_comp;
default:
- goto done;
+ send_status = IB_WC_LOC_QP_OP_ERR;
+ goto serr;
}
sge = &sqp->s_sge.sge;
@@ -448,8 +428,7 @@ again:
sqp->s_len -= len;
}
- if (wqe->wr.opcode == IB_WR_RDMA_WRITE ||
- wqe->wr.opcode == IB_WR_RDMA_READ)
+ if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))
goto send_comp;
if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)
@@ -458,32 +437,89 @@ again:
wc.opcode = IB_WC_RECV;
wc.wr_id = qp->r_wr_id;
wc.status = IB_WC_SUCCESS;
- wc.vendor_err = 0;
wc.byte_len = wqe->length;
wc.qp = &qp->ibqp;
wc.src_qp = qp->remote_qpn;
- wc.pkey_index = 0;
wc.slid = qp->remote_ah_attr.dlid;
wc.sl = qp->remote_ah_attr.sl;
- wc.dlid_path_bits = 0;
wc.port_num = 1;
/* Signal completion event if the solicited bit is set. */
ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
wqe->wr.send_flags & IB_SEND_SOLICITED);
send_comp:
+ spin_lock_irqsave(&sqp->s_lock, flags);
+flush_send:
sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
- ipath_send_complete(sqp, wqe, IB_WC_SUCCESS);
+ ipath_send_complete(sqp, wqe, send_status);
goto again;
+rnr_nak:
+ /* Handle RNR NAK */
+ if (qp->ibqp.qp_type == IB_QPT_UC)
+ goto send_comp;
+ /*
+ * Note: we don't need the s_lock held since the BUSY flag
+ * makes this single threaded.
+ */
+ if (sqp->s_rnr_retry == 0) {
+ send_status = IB_WC_RNR_RETRY_EXC_ERR;
+ goto serr;
+ }
+ if (sqp->s_rnr_retry_cnt < 7)
+ sqp->s_rnr_retry--;
+ spin_lock_irqsave(&sqp->s_lock, flags);
+ if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_RECV_OK))
+ goto clr_busy;
+ sqp->s_flags |= IPATH_S_WAITING;
+ dev->n_rnr_naks++;
+ sqp->s_rnr_timeout = ib_ipath_rnr_table[qp->r_min_rnr_timer];
+ ipath_insert_rnr_queue(sqp);
+ goto clr_busy;
+
+inv_err:
+ send_status = IB_WC_REM_INV_REQ_ERR;
+ wc.status = IB_WC_LOC_QP_OP_ERR;
+ goto err;
+
+acc_err:
+ send_status = IB_WC_REM_ACCESS_ERR;
+ wc.status = IB_WC_LOC_PROT_ERR;
+err:
+ /* responder goes to error state */
+ ipath_rc_error(qp, wc.status);
+
+serr:
+ spin_lock_irqsave(&sqp->s_lock, flags);
+ ipath_send_complete(sqp, wqe, send_status);
+ if (sqp->ibqp.qp_type == IB_QPT_RC) {
+ int lastwqe = ipath_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
+
+ sqp->s_flags &= ~IPATH_S_BUSY;
+ spin_unlock_irqrestore(&sqp->s_lock, flags);
+ if (lastwqe) {
+ struct ib_event ev;
+
+ ev.device = sqp->ibqp.device;
+ ev.element.qp = &sqp->ibqp;
+ ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
+ sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context);
+ }
+ goto done;
+ }
+clr_busy:
+ sqp->s_flags &= ~IPATH_S_BUSY;
+unlock:
+ spin_unlock_irqrestore(&sqp->s_lock, flags);
done:
- if (atomic_dec_and_test(&qp->refcount))
+ if (qp && atomic_dec_and_test(&qp->refcount))
wake_up(&qp->wait);
}
-static void want_buffer(struct ipath_devdata *dd)
+static void want_buffer(struct ipath_devdata *dd, struct ipath_qp *qp)
{
- if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA)) {
+ if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA) ||
+ qp->ibqp.qp_type == IB_QPT_SMI) {
unsigned long flags;
spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
@@ -501,26 +537,36 @@ static void want_buffer(struct ipath_devdata *dd)
* @dev: the device we ran out of buffers on
*
* Called when we run out of PIO buffers.
+ * If we are now in the error state, return zero to flush the
+ * send work request.
*/
-static void ipath_no_bufs_available(struct ipath_qp *qp,
+static int ipath_no_bufs_available(struct ipath_qp *qp,
struct ipath_ibdev *dev)
{
unsigned long flags;
+ int ret = 1;
/*
* Note that as soon as want_buffer() is called and
* possibly before it returns, ipath_ib_piobufavail()
- * could be called. If we are still in the tasklet function,
- * tasklet_hi_schedule() will not call us until the next time
- * tasklet_hi_schedule() is called.
- * We leave the busy flag set so that another post send doesn't
- * try to put the same QP on the piowait list again.
+ * could be called. Therefore, put QP on the piowait list before
+ * enabling the PIO avail interrupt.
*/
- spin_lock_irqsave(&dev->pending_lock, flags);
- list_add_tail(&qp->piowait, &dev->piowait);
- spin_unlock_irqrestore(&dev->pending_lock, flags);
- want_buffer(dev->dd);
- dev->n_piowait++;
+ spin_lock_irqsave(&qp->s_lock, flags);
+ if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) {
+ dev->n_piowait++;
+ qp->s_flags |= IPATH_S_WAITING;
+ qp->s_flags &= ~IPATH_S_BUSY;
+ spin_lock(&dev->pending_lock);
+ if (list_empty(&qp->piowait))
+ list_add_tail(&qp->piowait, &dev->piowait);
+ spin_unlock(&dev->pending_lock);
+ } else
+ ret = 0;
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+ if (ret)
+ want_buffer(dev->dd, qp);
+ return ret;
}
/**
@@ -596,15 +642,13 @@ void ipath_do_send(unsigned long data)
struct ipath_qp *qp = (struct ipath_qp *)data;
struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
int (*make_req)(struct ipath_qp *qp);
-
- if (test_and_set_bit(IPATH_S_BUSY, &qp->s_busy))
- goto bail;
+ unsigned long flags;
if ((qp->ibqp.qp_type == IB_QPT_RC ||
qp->ibqp.qp_type == IB_QPT_UC) &&
qp->remote_ah_attr.dlid == dev->dd->ipath_lid) {
ipath_ruc_loopback(qp);
- goto clear;
+ goto bail;
}
if (qp->ibqp.qp_type == IB_QPT_RC)
@@ -614,6 +658,19 @@ void ipath_do_send(unsigned long data)
else
make_req = ipath_make_ud_req;
+ spin_lock_irqsave(&qp->s_lock, flags);
+
+ /* Return if we are already busy processing a work request. */
+ if ((qp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) ||
+ !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) {
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+ goto bail;
+ }
+
+ qp->s_flags |= IPATH_S_BUSY;
+
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+
again:
/* Check for a constructed packet to be sent. */
if (qp->s_hdrwords != 0) {
@@ -623,8 +680,8 @@ again:
*/
if (ipath_verbs_send(qp, &qp->s_hdr, qp->s_hdrwords,
qp->s_cur_sge, qp->s_cur_size)) {
- ipath_no_bufs_available(qp, dev);
- goto bail;
+ if (ipath_no_bufs_available(qp, dev))
+ goto bail;
}
dev->n_unicast_xmit++;
/* Record that we sent the packet and s_hdr is empty. */
@@ -633,16 +690,20 @@ again:
if (make_req(qp))
goto again;
-clear:
- clear_bit(IPATH_S_BUSY, &qp->s_busy);
+
bail:;
}
+/*
+ * This should be called with s_lock held.
+ */
void ipath_send_complete(struct ipath_qp *qp, struct ipath_swqe *wqe,
enum ib_wc_status status)
{
- unsigned long flags;
- u32 last;
+ u32 old_last, last;
+
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND))
+ return;
/* See ch. 11.2.4.1 and 10.7.3.1 */
if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||
@@ -650,27 +711,25 @@ void ipath_send_complete(struct ipath_qp *qp, struct ipath_swqe *wqe,
status != IB_WC_SUCCESS) {
struct ib_wc wc;
+ memset(&wc, 0, sizeof wc);
wc.wr_id = wqe->wr.wr_id;
wc.status = status;
wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
- wc.vendor_err = 0;
- wc.byte_len = wqe->length;
- wc.imm_data = 0;
wc.qp = &qp->ibqp;
- wc.src_qp = 0;
- wc.wc_flags = 0;
- wc.pkey_index = 0;
- wc.slid = 0;
- wc.sl = 0;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
- ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
+ if (status == IB_WC_SUCCESS)
+ wc.byte_len = wqe->length;
+ ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc,
+ status != IB_WC_SUCCESS);
}
- spin_lock_irqsave(&qp->s_lock, flags);
- last = qp->s_last;
+ old_last = last = qp->s_last;
if (++last >= qp->s_size)
last = 0;
qp->s_last = last;
- spin_unlock_irqrestore(&qp->s_lock, flags);
+ if (qp->s_cur == old_last)
+ qp->s_cur = last;
+ if (qp->s_tail == old_last)
+ qp->s_tail = last;
+ if (qp->state == IB_QPS_SQD && last == qp->s_cur)
+ qp->s_draining = 0;
}
diff --git a/drivers/infiniband/hw/ipath/ipath_sd7220.c b/drivers/infiniband/hw/ipath/ipath_sd7220.c
index 81808ba..c0c9b44 100644
--- a/drivers/infiniband/hw/ipath/ipath_sd7220.c
+++ b/drivers/infiniband/hw/ipath/ipath_sd7220.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -68,28 +68,321 @@
#define EPB_IB_QUAD0_CS (1U << EPB_IB_QUAD0_CS_SHF)
#define EPB_IB_UC_CS_SHF (26)
#define EPB_PCIE_UC_CS_SHF (27)
+#define EPB_GLOBAL_WR (1U << (EPB_ADDR_SHF + 8))
/* Forward declarations. */
static int ipath_sd7220_reg_mod(struct ipath_devdata *dd, int sdnum, u32 loc,
u32 data, u32 mask);
+static int ibsd_mod_allchnls(struct ipath_devdata *dd, int loc, int val,
+ int mask);
static int ipath_sd_trimdone_poll(struct ipath_devdata *dd);
+static void ipath_sd_trimdone_monitor(struct ipath_devdata *dd,
+ const char *where);
static int ipath_sd_setvals(struct ipath_devdata *dd);
static int ipath_sd_early(struct ipath_devdata *dd);
static int ipath_sd_dactrim(struct ipath_devdata *dd);
/* Set the registers that IBC may muck with to their default "preset" values */
int ipath_sd7220_presets(struct ipath_devdata *dd);
+static int ipath_internal_presets(struct ipath_devdata *dd);
/* Tweak the register (CMUCTRL5) that contains the TRIMSELF controls */
static int ipath_sd_trimself(struct ipath_devdata *dd, int val);
+static int epb_access(struct ipath_devdata *dd, int sdnum, int claim);
+
+void ipath_set_relock_poll(struct ipath_devdata *dd, int ibup);
+
+/*
+ * Below keeps track of whether the "once per power-on" initialization has
+ * been done, because uC code Version 1.32.17 or higher allows the uC to
+ * be reset at will, and Automatic Equalization may requore it. So the
+ * state of the reset "pin", as reflected in was_reset parameter to
+ * ipath_sd7220_init() is no longer valid. Instead, we check for the
+ * actual uC code having been loaded.
+ */
+static int ipath_ibsd_ucode_loaded(struct ipath_devdata *dd)
+{
+ if (!dd->serdes_first_init_done && (ipath_sd7220_ib_vfy(dd) > 0))
+ dd->serdes_first_init_done = 1;
+ return dd->serdes_first_init_done;
+}
+
+/* repeat #define for local use. "Real" #define is in ipath_iba7220.c */
+#define INFINIPATH_HWE_IB_UC_MEMORYPARITYERR 0x0000004000000000ULL
+#define IB_MPREG5 (EPB_LOC(6, 0, 0xE) | (1L << EPB_IB_UC_CS_SHF))
+#define IB_MPREG6 (EPB_LOC(6, 0, 0xF) | (1U << EPB_IB_UC_CS_SHF))
+#define UC_PAR_CLR_D 8
+#define UC_PAR_CLR_M 0xC
+#define IB_CTRL2(chn) (EPB_LOC(chn, 7, 3) | EPB_IB_QUAD0_CS)
+#define START_EQ1(chan) EPB_LOC(chan, 7, 0x27)
+
+void ipath_sd7220_clr_ibpar(struct ipath_devdata *dd)
+{
+ int ret;
+
+ /* clear, then re-enable parity errs */
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_MPREG6,
+ UC_PAR_CLR_D, UC_PAR_CLR_M);
+ if (ret < 0) {
+ ipath_dev_err(dd, "Failed clearing IBSerDes Parity err\n");
+ goto bail;
+ }
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_MPREG6, 0,
+ UC_PAR_CLR_M);
+
+ ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
+ udelay(4);
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear,
+ INFINIPATH_HWE_IB_UC_MEMORYPARITYERR);
+ ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
+bail:
+ return;
+}
+
+/* After a reset or other unusual event, the epb interface may need
+ * to be re-synchronized, between the host and the uC.
+ * returns <0 for failure
+ * (which can only happen if we fail IBSD_RESYNC_TRIES times)
+ */
+#define IBSD_RESYNC_TRIES 3
+#define IB_PGUDP(chn) (EPB_LOC((chn), 2, 1) | EPB_IB_QUAD0_CS)
+#define IB_CMUDONE(chn) (EPB_LOC((chn), 7, 0xF) | EPB_IB_QUAD0_CS)
+
+static int ipath_resync_ibepb(struct ipath_devdata *dd)
+{
+ int ret, pat, tries, chn;
+ u32 loc;
+
+ ret = -1;
+ chn = 0;
+ for (tries = 0; tries < (4 * IBSD_RESYNC_TRIES); ++tries) {
+ loc = IB_PGUDP(chn);
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, 0, 0);
+ if (ret < 0) {
+ ipath_dev_err(dd, "Failed read in resync\n");
+ continue;
+ }
+ if (ret != 0xF0 && ret != 0x55 && tries == 0)
+ ipath_dev_err(dd, "unexpected pattern in resync\n");
+ pat = ret ^ 0xA5; /* alternate F0 and 55 */
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, pat, 0xFF);
+ if (ret < 0) {
+ ipath_dev_err(dd, "Failed write in resync\n");
+ continue;
+ }
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, 0, 0);
+ if (ret < 0) {
+ ipath_dev_err(dd, "Failed re-read in resync\n");
+ continue;
+ }
+ if (ret != pat) {
+ ipath_dev_err(dd, "Failed compare1 in resync\n");
+ continue;
+ }
+ loc = IB_CMUDONE(chn);
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, 0, 0);
+ if (ret < 0) {
+ ipath_dev_err(dd, "Failed CMUDONE rd in resync\n");
+ continue;
+ }
+ if ((ret & 0x70) != ((chn << 4) | 0x40)) {
+ ipath_dev_err(dd, "Bad CMUDONE value %02X, chn %d\n",
+ ret, chn);
+ continue;
+ }
+ if (++chn == 4)
+ break; /* Success */
+ }
+ ipath_cdbg(VERBOSE, "Resync in %d tries\n", tries);
+ return (ret > 0) ? 0 : ret;
+}
+
+/*
+ * Localize the stuff that should be done to change IB uC reset
+ * returns <0 for errors.
+ */
+static int ipath_ibsd_reset(struct ipath_devdata *dd, int assert_rst)
+{
+ u64 rst_val;
+ int ret = 0;
+ unsigned long flags;
+
+ rst_val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibserdesctrl);
+ if (assert_rst) {
+ /*
+ * Vendor recommends "interrupting" uC before reset, to
+ * minimize possible glitches.
+ */
+ spin_lock_irqsave(&dd->ipath_sdepb_lock, flags);
+ epb_access(dd, IB_7220_SERDES, 1);
+ rst_val |= 1ULL;
+ /* Squelch possible parity error from _asserting_ reset */
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask,
+ dd->ipath_hwerrmask &
+ ~INFINIPATH_HWE_IB_UC_MEMORYPARITYERR);
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_ibserdesctrl, rst_val);
+ /* flush write, delay to ensure it took effect */
+ ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
+ udelay(2);
+ /* once it's reset, can remove interrupt */
+ epb_access(dd, IB_7220_SERDES, -1);
+ spin_unlock_irqrestore(&dd->ipath_sdepb_lock, flags);
+ } else {
+ /*
+ * Before we de-assert reset, we need to deal with
+ * possible glitch on the Parity-error line.
+ * Suppress it around the reset, both in chip-level
+ * hwerrmask and in IB uC control reg. uC will allow
+ * it again during startup.
+ */
+ u64 val;
+ rst_val &= ~(1ULL);
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask,
+ dd->ipath_hwerrmask &
+ ~INFINIPATH_HWE_IB_UC_MEMORYPARITYERR);
+
+ ret = ipath_resync_ibepb(dd);
+ if (ret < 0)
+ ipath_dev_err(dd, "unable to re-sync IB EPB\n");
+
+ /* set uC control regs to suppress parity errs */
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_MPREG5, 1, 1);
+ if (ret < 0)
+ goto bail;
+ /* IB uC code past Version 1.32.17 allow suppression of wdog */
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_MPREG6, 0x80,
+ 0x80);
+ if (ret < 0) {
+ ipath_dev_err(dd, "Failed to set WDOG disable\n");
+ goto bail;
+ }
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_ibserdesctrl, rst_val);
+ /* flush write, delay for startup */
+ ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
+ udelay(1);
+ /* clear, then re-enable parity errs */
+ ipath_sd7220_clr_ibpar(dd);
+ val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_hwerrstatus);
+ if (val & INFINIPATH_HWE_IB_UC_MEMORYPARITYERR) {
+ ipath_dev_err(dd, "IBUC Parity still set after RST\n");
+ dd->ipath_hwerrmask &=
+ ~INFINIPATH_HWE_IB_UC_MEMORYPARITYERR;
+ }
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask,
+ dd->ipath_hwerrmask);
+ }
+
+bail:
+ return ret;
+}
+
+static void ipath_sd_trimdone_monitor(struct ipath_devdata *dd,
+ const char *where)
+{
+ int ret, chn, baduns;
+ u64 val;
+
+ if (!where)
+ where = "?";
+
+ /* give time for reset to settle out in EPB */
+ udelay(2);
+
+ ret = ipath_resync_ibepb(dd);
+ if (ret < 0)
+ ipath_dev_err(dd, "not able to re-sync IB EPB (%s)\n", where);
+
+ /* Do "sacrificial read" to get EPB in sane state after reset */
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_CTRL2(0), 0, 0);
+ if (ret < 0)
+ ipath_dev_err(dd, "Failed TRIMDONE 1st read, (%s)\n", where);
+
+ /* Check/show "summary" Trim-done bit in IBCStatus */
+ val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
+ if (val & (1ULL << 11))
+ ipath_cdbg(VERBOSE, "IBCS TRIMDONE set (%s)\n", where);
+ else
+ ipath_dev_err(dd, "IBCS TRIMDONE clear (%s)\n", where);
+ /*
+ * Do "dummy read/mod/wr" to get EPB in sane state after reset
+ * The default (and hopefully only, D6..0) value for MPREG6 is 0, and
+ * we want to set to 0x80. Since we can't trust read, or we wouldn't
+ * be doing this, hope for the best
+ */
+ udelay(2);
+
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_MPREG6, 0x80, 0x80);
+ if (ret < 0)
+ ipath_dev_err(dd, "Failed Dummy RMW, (%s)\n", where);
+ udelay(10);
+
+ baduns = 0;
+
+ for (chn = 3; chn >= 0; --chn) {
+ /* Read CTRL reg for each channel to check TRIMDONE */
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
+ IB_CTRL2(chn), 0, 0);
+ if (ret < 0)
+ ipath_dev_err(dd, "Failed checking TRIMDONE, chn %d"
+ " (%s)\n", chn, where);
+
+ if (!(ret & 0x10)) {
+ int probe;
+ baduns |= (1 << chn);
+ ipath_dev_err(dd, "TRIMDONE cleared on chn %d (%02X)."
+ " (%s)\n", chn, ret, where);
+ probe = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
+ IB_PGUDP(0), 0, 0);
+ ipath_dev_err(dd, "probe is %d (%02X)\n",
+ probe, probe);
+ probe = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
+ IB_CTRL2(chn), 0, 0);
+ ipath_dev_err(dd, "re-read: %d (%02X)\n",
+ probe, probe);
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
+ IB_CTRL2(chn), 0x10, 0x10);
+ if (ret < 0)
+ ipath_dev_err(dd,
+ "Err on TRIMDONE rewrite1\n");
+ }
+ }
+ for (chn = 3; chn >= 0; --chn) {
+ /* Read CTRL reg for each channel to check TRIMDONE */
+ if (baduns & (1 << chn)) {
+ ipath_dev_err(dd,
+ "Reseting TRIMDONE on chn %d (%s)\n",
+ chn, where);
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
+ IB_CTRL2(chn), 0x10, 0x10);
+ if (ret < 0)
+ ipath_dev_err(dd, "Failed re-setting "
+ "TRIMDONE, chn %d (%s)\n",
+ chn, where);
+ }
+ }
+}
/*
* Below is portion of IBA7220-specific bringup_serdes() that actually
* deals with registers and memory within the SerDes itself.
+ * Post IB uC code version 1.32.17, was_reset being 1 is not really
+ * informative, so we double-check.
*/
int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset)
{
- u64 val;
- int ret = 0;
+ int ret = 1; /* default to failure */
+ int first_reset;
+ if (!was_reset) {
+ /* entered with reset not asserted, we need to do it */
+ ipath_ibsd_reset(dd, 1);
+ ipath_sd_trimdone_monitor(dd, "Driver-reload");
+ }
+ /* Substitute our deduced value for was_reset */
+ ret = ipath_ibsd_ucode_loaded(dd);
+ if (ret < 0) {
+ ret = 1;
+ goto done;
+ }
+ first_reset = !ret; /* First reset if IBSD uCode not yet loaded */
do {
/*
* Alter some regs per vendor latest doc, reset-defaults
@@ -105,9 +398,8 @@ int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset)
/* Set DAC manual trim IB.
* We only do this once after chip has been reset (usually
* same as once per system boot).
- * Relies on quiet_serdes() not putting uC in reset.
*/
- if (was_reset) {
+ if (first_reset) {
ret = ipath_sd_dactrim(dd);
if (ret < 0) {
ipath_dev_err(dd,
@@ -116,12 +408,13 @@ int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset)
break;
}
}
- /* Set various registers (DDS and RXEQ) that will be
- * controlled by IBC (in 1.2 mode) to reasonable
- * preset values; set needed to force, because init
+ /*
+ * Set various registers (DDS and RXEQ) that will be
+ * controlled by IBC (in 1.2 mode) to reasonable preset values
+ * Calling the "internal" version avoids the "check for needed"
+ * and "trimdone monitor" that might be counter-productive.
*/
- dd->ipath_presets_needed = 1;
- ret = ipath_sd7220_presets(dd);
+ ret = ipath_internal_presets(dd);
if (ret < 0) {
ipath_dev_err(dd, "Failed to set IB SERDES presets\n");
ret = 1;
@@ -136,7 +429,7 @@ int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset)
/* Load image, then try to verify */
ret = 0; /* Assume success */
- if (was_reset) {
+ if (first_reset) {
ipath_dbg("SerDes uC was reset, reloading PRAM\n");
ret = ipath_sd7220_ib_load(dd);
if (ret < 0) {
@@ -160,22 +453,53 @@ int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset)
ret = 0;
} /* end if verified */
} /* end if loaded */
- } /* end if was_reset */
+ } /* end if first_reset */
} while (0) ; /* do_while for goto-less bailing */
- if (ret == 0) {
- /* Prev steps all worked, continue bringup */
- /* De-assert RESET to uC */
+ if (ret == 0 && first_reset) {
+ /*
+ * Prev steps all worked, continue bringup
+ * De-assert RESET to uC, only in first reset, to allow
+ * trimming.
+ */
int trim_done;
- val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibserdesctrl);
- val &= ~(1ULL);
- ipath_write_kreg(dd, dd->ipath_kregs->kr_ibserdesctrl, val);
+ /*
+ * Since our default setup sets START_EQ1 to
+ * PRESET, we need to clear that for this very first run.
+ */
+ ret = ibsd_mod_allchnls(dd, START_EQ1(0), 0, 0x38);
+ if (ret < 0) {
+ ipath_dev_err(dd, "Failed clearing START_EQ1\n");
+ ret = 1;
+ goto done;
+ }
+
+ ipath_ibsd_reset(dd, 0);
+ /*
+ * If this is not the first reset, trimdone should be set
+ * already. We may need to check about this.
+ */
trim_done = ipath_sd_trimdone_poll(dd);
+ /*
+ * Whether or not trimdone succeeded, we need to put the
+ * uC back into reset to avoid a possible fight with the
+ * IBC state-machine.
+ */
+ ipath_ibsd_reset(dd, 1);
+
if (!trim_done) {
ipath_dev_err(dd, "No TRIMDONE seen\n");
ret = 1;
+ goto done;
}
+ /*
+ * DEBUG: check each time we reset if trimdone bits have
+ * gotten cleared, and re-set them.
+ */
+ ipath_sd_trimdone_monitor(dd, "First-reset");
+ /* Remember so we do not re-do the load, dactrim, etc. */
+ dd->serdes_first_init_done = 1;
}
if (ret == 0) {
/*
@@ -188,6 +512,9 @@ int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset)
if (val_stat < 0)
ret = 1;
}
+done:
+ /* start relock timer regardless, but start at 1 second */
+ ipath_set_relock_poll(dd, -1);
return ret;
}
@@ -237,7 +564,7 @@ static int epb_access(struct ipath_devdata *dd, int sdnum, int claim)
accval = ipath_read_kreg32(dd, acc);
owned = !!(accval & EPB_ACC_GNT);
- if (owned && claim < 0) {
+ if (claim < 0) {
/* Need to release */
u64 pollval;
/*
@@ -255,7 +582,7 @@ static int epb_access(struct ipath_devdata *dd, int sdnum, int claim)
pollval = ipath_read_kreg32(dd, acc);
if (pollval & EPB_ACC_GNT)
owned = -1;
- } else if (!owned && claim > 0) {
+ } else if (claim > 0) {
/* Need to claim */
u64 pollval;
u64 newval = EPB_ACC_REQ | oct_sel;
@@ -717,6 +1044,11 @@ static struct dds_init {
#define RXEQ_SDR_G1CNT_Z1CNT 0x11
#define RXEQ_SDR_ZCNT 23
+/*
+ * The values below (as opposed to what "was") were experimentally determined
+ * to reduce IB Symbol errors, but currently all four "sets" are the same.
+ * with more experimentation, we will derive a range.
+ */
static struct rxeq_init {
u16 rdesc; /* in form used in SerDesDDSRXEQ */
u8 rdata[4];
@@ -724,23 +1056,20 @@ static struct rxeq_init {
/* Set Rcv Eq. to Preset node */
RXEQ_VAL_ALL(7, 0x27, 0x10),
/* Set DFELTHFDR/HDR thresholds */
- RXEQ_VAL(7, 8, 0, 1, 2, 3), /* FDR */
+ RXEQ_VAL(7, 8, 0, 0, 0, 0), /* FDR, was 0, 1, 2, 3 */
RXEQ_VAL(7, 0x21, 0, 0, 0, 0), /* HDR */
/* Set TLTHFDR/HDR theshold */
- RXEQ_VAL(7, 9, 0, 2, 4, 6), /* FDR */
- RXEQ_VAL(7, 0x23, 0, 1, 2, 3), /* HDR */
+ RXEQ_VAL(7, 9, 2, 2, 2, 2), /* FDR, was 0, 2, 4, 6 */
+ RXEQ_VAL(7, 0x23, 2, 2, 2, 2), /* HDR, was 0, 1, 2, 3 */
/* Set Preamp setting 2 (ZFR/ZCNT) */
- RXEQ_VAL(7, 0x1B, 12, 16, 20, 24), /* FDR */
- RXEQ_VAL(7, 0x1C, 12, 16, 20, 24), /* HDR */
+ RXEQ_VAL(7, 0x1B, 12, 12, 12, 12), /* FDR, was 12, 16, 20, 24 */
+ RXEQ_VAL(7, 0x1C, 12, 12, 12, 12), /* HDR, was 12, 16, 20, 24 */
/* Set Preamp DC gain and Setting 1 (GFR/GHR) */
- RXEQ_VAL(7, 0x1E, 0x10, 0x11, 0x12, 0x14), /* FDR */
- RXEQ_VAL(7, 0x1F, 0x10, 0x11, 0x12, 0x14), /* HDR */
+ RXEQ_VAL(7, 0x1E, 0x10, 0x10, 0x10, 0x10), /* FDR, was 0x10, 0x11, 0x12, 0x14 */
+ RXEQ_VAL(7, 0x1F, 0x10, 0x10, 0x10, 0x10), /* HDR, was 0x10, 0x11, 0x12, 0x14 */
/* Toggle RELOCK (in VCDL_CTRL0) to lock to data */
RXEQ_VAL_ALL(6, 6, 0x20), /* Set D5 High */
RXEQ_VAL_ALL(6, 6, 0), /* Set D5 Low */
- /* Enable parallel output data and clock, RCLKRLS */
- RXEQ_VAL_ALL(0, 2, 0x50), /* Set D7 Low */
- RXEQ_VAL_ALL(0, 2, 0xD0) /* Set D7 High */
};
/* There are 17 values from vendor, but IBC only accesses the first 16 */
@@ -821,8 +1150,8 @@ static int ipath_sd_setvals(struct ipath_devdata *dd)
#define CMUCTRL5 EPB_LOC(7, 0, 0x15)
#define RXHSCTRL0(chan) EPB_LOC(chan, 6, 0)
#define VCDL_DAC2(chan) EPB_LOC(chan, 6, 5)
+#define VCDL_CTRL0(chan) EPB_LOC(chan, 6, 6)
#define VCDL_CTRL2(chan) EPB_LOC(chan, 6, 8)
-#define START_EQ1(chan) EPB_LOC(chan, 7, 0x27)
#define START_EQ2(chan) EPB_LOC(chan, 7, 0x28)
static int ibsd_sto_noisy(struct ipath_devdata *dd, int loc, int val, int mask)
@@ -839,10 +1168,6 @@ static int ibsd_sto_noisy(struct ipath_devdata *dd, int loc, int val, int mask)
" addr 0x%X, chnl %d, val 0x%02X, mask 0x%02X\n",
(sloc & 0xF), (sloc >> 9) & 0x3f, (sloc >> 4) & 7,
val & 0xFF, mask & 0xFF);
- else
- ipath_cdbg(VERBOSE, "IBSD(%d,0x%02X,%d) <- 0x%02X (%02X)\n",
- (sloc & 0xF), (sloc >> 9) & 0x3f, (sloc >> 4) & 7,
- val & 0xFF, mask & 0xFF);
return ret;
}
@@ -860,6 +1185,36 @@ static int ibsd_mod_allchnls(struct ipath_devdata *dd, int loc, int val,
int ret = -1;
int chnl;
+ if (loc & EPB_GLOBAL_WR) {
+ /* our caller has assured us that we can set all four
+ * channels at once. Trust that. If mask is not 0xFF,
+ * we will read the _specified_ channel for our starting
+ * value.
+ */
+ loc |= (1U << EPB_IB_QUAD0_CS_SHF);
+ chnl = (loc >> (4 + EPB_ADDR_SHF)) & 7;
+ if (mask != 0xFF) {
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
+ loc & ~EPB_GLOBAL_WR, 0, 0);
+ if (ret < 0) {
+ int sloc = loc >> EPB_ADDR_SHF;
+ ipath_dev_err(dd, "pre-read failed: elt %d,"
+ " addr 0x%X, chnl %d\n", (sloc & 0xF),
+ (sloc >> 9) & 0x3f, chnl);
+ return ret;
+ }
+ val = (ret & ~mask) | (val & mask);
+ }
+ loc &= ~(7 << (4+EPB_ADDR_SHF));
+ ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, val, 0xFF);
+ if (ret < 0) {
+ int sloc = loc >> EPB_ADDR_SHF;
+ ipath_dev_err(dd, "Global WR failed: elt %d,"
+ " addr 0x%X, val %02X\n",
+ (sloc & 0xF), (sloc >> 9) & 0x3f, val);
+ }
+ return ret;
+ }
/* Clear "channel" and set CS so we can simply iterate */
loc &= ~(7 << (4+EPB_ADDR_SHF));
loc |= (1U << EPB_IB_QUAD0_CS_SHF);
@@ -875,12 +1230,6 @@ static int ibsd_mod_allchnls(struct ipath_devdata *dd, int loc, int val,
(sloc & 0xF), (sloc >> 9) & 0x3f, chnl,
val & 0xFF, mask & 0xFF);
break;
- } else {
- int sloc = loc >> EPB_ADDR_SHF;
- ipath_cdbg(VERBOSE,
- "IBSD(%d,0x%02X,%d) <- 0x%02X (%02X)\n",
- (sloc & 0xF), (sloc >> 9) & 0x3f, chnl,
- val & 0xFF, mask & 0xFF);
}
}
return ret;
@@ -933,28 +1282,51 @@ static int set_rxeq_vals(struct ipath_devdata *dd, int vsel)
return ret;
}
-/* Set the default values (row 0) for DDR Driver Demphasis.
+/*
+ * Set the default values (row 0) for DDR Driver Demphasis.
* we do this initially and whenever we turn off IB-1.2
* Vendor recommends non-default presets, depending on
* cable length. Initial testing will assume 3 meter cables.
+ * The "default" values for Rx equalization are also stored to
+ * SerDes registers. Formerly (and still default), we used set 2.
+ * For experimenting with cables and link-partners, we allow changing
+ * that via a module parameter.
*/
-int ipath_sd7220_presets(struct ipath_devdata *dd)
+static unsigned ipath_rxeq_set = 2;
+module_param_named(rxeq_default_set, ipath_rxeq_set, uint,
+ S_IWUSR | S_IRUGO);
+MODULE_PARM_DESC(rxeq_default_set, "Which set [0..3] of Rx Equalization values is default");
+
+static int ipath_internal_presets(struct ipath_devdata *dd)
{
int ret = 0;
- if (!dd->ipath_presets_needed)
- return ret;
- dd->ipath_presets_needed = 0;
ret = set_dds_vals(dd, dds_init_vals + DDS_3M);
if (ret < 0)
ipath_dev_err(dd, "Failed to set default DDS values\n");
- ret = set_rxeq_vals(dd, 2);
+ ret = set_rxeq_vals(dd, ipath_rxeq_set & 3);
if (ret < 0)
ipath_dev_err(dd, "Failed to set default RXEQ values\n");
return ret;
}
+int ipath_sd7220_presets(struct ipath_devdata *dd)
+{
+ int ret = 0;
+
+ if (!dd->ipath_presets_needed)
+ return ret;
+ dd->ipath_presets_needed = 0;
+ /* Assert uC reset, so we don't clash with it. */
+ ipath_ibsd_reset(dd, 1);
+ udelay(2);
+ ipath_sd_trimdone_monitor(dd, "link-down");
+
+ ret = ipath_internal_presets(dd);
+return ret;
+}
+
static int ipath_sd_trimself(struct ipath_devdata *dd, int val)
{
return ibsd_sto_noisy(dd, CMUCTRL5, val, 0xFF);
@@ -1054,30 +1426,130 @@ static int ipath_sd_dactrim(struct ipath_devdata *dd)
return ret;
}
+#define RELOCK_FIRST_MS 3
+#define RXLSPPM(chan) EPB_LOC(chan, 0, 2)
+void ipath_toggle_rclkrls(struct ipath_devdata *dd)
+{
+ int loc = RXLSPPM(0) | EPB_GLOBAL_WR;
+ int ret;
+
+ ret = ibsd_mod_allchnls(dd, loc, 0, 0x80);
+ if (ret < 0)
+ ipath_dev_err(dd, "RCLKRLS failed to clear D7\n");
+ else {
+ udelay(1);
+ ibsd_mod_allchnls(dd, loc, 0x80, 0x80);
+ }
+ /* And again for good measure */
+ udelay(1);
+ ret = ibsd_mod_allchnls(dd, loc, 0, 0x80);
+ if (ret < 0)
+ ipath_dev_err(dd, "RCLKRLS failed to clear D7\n");
+ else {
+ udelay(1);
+ ibsd_mod_allchnls(dd, loc, 0x80, 0x80);
+ }
+ /* Now reset xgxs and IBC to complete the recovery */
+ dd->ipath_f_xgxs_reset(dd);
+}
+
/*
- * ipath_sd7220_enable_aeq - Enable Auto-Equalization.
- * @dd: the infinipath device
- *
- * Function to turn on adaptive equalization. On IBA7220, the only chip
- * so far to support it, we do this by setting START_EQ1 (elt 7 reg 0x27)
- * PRESET to zero and STEADY_STATE to 1 (bits 4:3 = 01)
- * Returns <0 for errors, 0 for "not enabled", >0 for "enabled"
+ * Shut down the timer that polls for relock occasions, if needed
+ * this is "hooked" from ipath_7220_quiet_serdes(), which is called
+ * just before ipath_shutdown_device() in ipath_driver.c shuts down all
+ * the other timers
*/
-int ipath_sd7220_enable_aeq(struct ipath_devdata *dd)
+void ipath_shutdown_relock_poll(struct ipath_devdata *dd)
{
- int regval, chnl;
+ struct ipath_relock *irp = &dd->ipath_relock_singleton;
+ if (atomic_read(&irp->ipath_relock_timer_active)) {
+ del_timer_sync(&irp->ipath_relock_timer);
+ atomic_set(&irp->ipath_relock_timer_active, 0);
+ }
+}
- if (dd->ipath_presets_needed)
- return 1;
- dd->ipath_presets_needed = 1;
- /* Clear PRESET and set STEADY_STATE, for all channels */
- for (chnl = 0; chnl < 4; ++chnl) {
- regval = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
- START_EQ1(chnl) | EPB_IB_QUAD0_CS,
- 8, 0x18);
- if (regval < 0)
- return regval;
+static unsigned ipath_relock_by_timer = 1;
+module_param_named(relock_by_timer, ipath_relock_by_timer, uint,
+ S_IWUSR | S_IRUGO);
+MODULE_PARM_DESC(relock_by_timer, "Allow relock attempt if link not up");
+
+static void ipath_run_relock(unsigned long opaque)
+{
+ struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
+ struct ipath_relock *irp = &dd->ipath_relock_singleton;
+ u64 val, ltstate;
+
+ if (!(dd->ipath_flags & IPATH_INITTED)) {
+ /* Not yet up, just reenable the timer for later */
+ irp->ipath_relock_interval = HZ;
+ mod_timer(&irp->ipath_relock_timer, jiffies + HZ);
+ return;
+ }
+
+ /*
+ * Check link-training state for "stuck" state.
+ * if found, try relock and schedule another try at
+ * exponentially growing delay, maxed at one second.
+ * if not stuck, our work is done.
+ */
+ val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
+ ltstate = ipath_ib_linktrstate(dd, val);
+
+ /* Below check was <= CFGDEBOUNCE, JBR requests change for test */
+ if (ltstate <= INFINIPATH_IBCS_LT_STATE_CFGWAITRMT
+ && ltstate != INFINIPATH_IBCS_LT_STATE_LINKUP) {
+ int timeoff;
+ /* Not up yet. Try again, if allowed by module-param */
+ if (ipath_relock_by_timer) {
+ if (dd->ipath_flags & IPATH_IB_AUTONEG_INPROG) {
+ ipath_cdbg(VERBOSE, "Skip RELOCK in AUTONEG\n");
+ } else if (!(dd->ipath_flags &
+ IPATH_IB_LINK_DISABLED)) {
+ ipath_cdbg(VERBOSE, "RELOCK\n");
+ ipath_toggle_rclkrls(dd);
+
+ }
+ }
+ /* re-set timer for next check */
+ timeoff = irp->ipath_relock_interval << 1;
+ if (timeoff > HZ)
+ timeoff = HZ;
+ irp->ipath_relock_interval = timeoff;
+
+ mod_timer(&irp->ipath_relock_timer, jiffies + timeoff);
+ } else {
+ /* Up, so no more need to check so often */
+ mod_timer(&irp->ipath_relock_timer, jiffies + HZ);
+ }
+}
+
+void ipath_set_relock_poll(struct ipath_devdata *dd, int ibup)
+{
+ struct ipath_relock *irp = &dd->ipath_relock_singleton;
+
+ if (ibup > 0) {
+ /* we are now up, so squelch timer */
+ if (atomic_read(&irp->ipath_relock_timer_active))
+ mod_timer(&irp->ipath_relock_timer, jiffies + HZ);
+ } else {
+ /* Transition to down, (re-)set timer to short interval. */
+ int timeout;
+ timeout = (HZ * ((ibup == -1) ? 1000 : RELOCK_FIRST_MS))/1000;
+ if (timeout == 0)
+ timeout = 1;
+ /* If timer has not yet been started, do so. */
+ if (atomic_inc_return(&irp->ipath_relock_timer_active) == 1) {
+ init_timer(&irp->ipath_relock_timer);
+ irp->ipath_relock_timer.function = ipath_run_relock;
+ irp->ipath_relock_timer.data = (unsigned long) dd;
+ irp->ipath_relock_interval = timeout;
+ irp->ipath_relock_timer.expires = jiffies + timeout;
+ add_timer(&irp->ipath_relock_timer);
+ } else {
+ irp->ipath_relock_interval = timeout;
+ mod_timer(&irp->ipath_relock_timer, jiffies + timeout);
+ atomic_dec(&irp->ipath_relock_timer_active);
+ }
}
- return 1;
}
diff --git a/drivers/infiniband/hw/ipath/ipath_sd7220_img.c b/drivers/infiniband/hw/ipath/ipath_sd7220_img.c
index be3b001..5ef59da 100644
--- a/drivers/infiniband/hw/ipath/ipath_sd7220_img.c
+++ b/drivers/infiniband/hw/ipath/ipath_sd7220_img.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -43,992 +43,992 @@
#include "ipath_7220.h"
static unsigned char ipath_sd7220_ib_img[] = {
-/*0000*/0x02, 0x0B, 0x08, 0x02, 0x0B, 0x61, 0xE5, 0xE6,
+/*0000*/0x02, 0x0A, 0x29, 0x02, 0x0A, 0x87, 0xE5, 0xE6,
0x30, 0xE6, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F,
/*0010*/0x00, 0xE5, 0xE2, 0x30, 0xE4, 0x04, 0x7E, 0x01,
0x80, 0x02, 0x7E, 0x00, 0xEE, 0x5F, 0x60, 0x08,
-/*0020*/0x53, 0xF9, 0xF7, 0xE4, 0xF5, 0xFE, 0x80, 0x0B,
- 0x7F, 0x0A, 0x12, 0x16, 0xB3, 0x53, 0xF9, 0xF7,
-/*0030*/0x75, 0xFE, 0x30, 0x75, 0xFC, 0x08, 0xE4, 0xF5,
- 0xFD, 0x43, 0xF9, 0x08, 0x22, 0x02, 0x00, 0x06,
-/*0040*/0x01, 0x1F, 0x10, 0x00, 0x04, 0x20, 0x00, 0x75,
+/*0020*/0x53, 0xF9, 0xF7, 0xE4, 0xF5, 0xFE, 0x80, 0x08,
+ 0x7F, 0x0A, 0x12, 0x17, 0x31, 0x12, 0x0E, 0xA2,
+/*0030*/0x75, 0xFC, 0x08, 0xE4, 0xF5, 0xFD, 0xE5, 0xE7,
+ 0x20, 0xE7, 0x03, 0x43, 0xF9, 0x08, 0x22, 0x00,
+/*0040*/0x01, 0x20, 0x11, 0x00, 0x04, 0x20, 0x00, 0x75,
0x51, 0x01, 0xE4, 0xF5, 0x52, 0xF5, 0x53, 0xF5,
-/*0050*/0x52, 0xF5, 0x7E, 0x7F, 0x04, 0x02, 0x04, 0x3D,
+/*0050*/0x52, 0xF5, 0x7E, 0x7F, 0x04, 0x02, 0x04, 0x38,
0xC2, 0x36, 0x05, 0x52, 0xE5, 0x52, 0xD3, 0x94,
/*0060*/0x0C, 0x40, 0x05, 0x75, 0x52, 0x01, 0xD2, 0x36,
0x90, 0x07, 0x0C, 0x74, 0x07, 0xF0, 0xA3, 0x74,
/*0070*/0xFF, 0xF0, 0xE4, 0xF5, 0x0C, 0xA3, 0xF0, 0x90,
0x07, 0x14, 0xF0, 0xA3, 0xF0, 0x75, 0x0B, 0x20,
/*0080*/0xF5, 0x09, 0xE4, 0xF5, 0x08, 0xE5, 0x08, 0xD3,
- 0x94, 0x30, 0x40, 0x03, 0x02, 0x04, 0x09, 0x12,
+ 0x94, 0x30, 0x40, 0x03, 0x02, 0x04, 0x04, 0x12,
/*0090*/0x00, 0x06, 0x15, 0x0B, 0xE5, 0x08, 0x70, 0x04,
0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x09,
/*00A0*/0x70, 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00,
- 0xEE, 0x5F, 0x60, 0x05, 0x12, 0x17, 0xF3, 0xD2,
+ 0xEE, 0x5F, 0x60, 0x05, 0x12, 0x18, 0x71, 0xD2,
/*00B0*/0x35, 0x53, 0xE1, 0xF7, 0xE5, 0x08, 0x45, 0x09,
0xFF, 0xE5, 0x0B, 0x25, 0xE0, 0x25, 0xE0, 0x24,
/*00C0*/0x83, 0xF5, 0x82, 0xE4, 0x34, 0x07, 0xF5, 0x83,
0xEF, 0xF0, 0x85, 0xE2, 0x20, 0xE5, 0x52, 0xD3,
-/*00D0*/0x94, 0x01, 0x40, 0x0D, 0x12, 0x1B, 0x83, 0xE0,
- 0x54, 0xA0, 0x64, 0x40, 0x70, 0x03, 0x02, 0x04,
-/*00E0*/0x00, 0x53, 0xF9, 0xF8, 0x90, 0x94, 0x70, 0xE4,
+/*00D0*/0x94, 0x01, 0x40, 0x0D, 0x12, 0x19, 0xF3, 0xE0,
+ 0x54, 0xA0, 0x64, 0x40, 0x70, 0x03, 0x02, 0x03,
+/*00E0*/0xFB, 0x53, 0xF9, 0xF8, 0x90, 0x94, 0x70, 0xE4,
0xF0, 0xE0, 0xF5, 0x10, 0xAF, 0x09, 0x12, 0x1E,
-/*00F0*/0x84, 0xAF, 0x08, 0xEF, 0x44, 0x08, 0xF5, 0x82,
+/*00F0*/0xB3, 0xAF, 0x08, 0xEF, 0x44, 0x08, 0xF5, 0x82,
0x75, 0x83, 0x80, 0xE0, 0xF5, 0x29, 0xEF, 0x44,
-/*0100*/0x07, 0xF5, 0x82, 0x75, 0x83, 0x9E, 0xE0, 0xF5,
- 0x22, 0x54, 0x40, 0xD3, 0x94, 0x00, 0x40, 0x1E,
-/*0110*/0xE5, 0x29, 0x54, 0xF0, 0x70, 0x21, 0x12, 0x1B,
- 0x83, 0xE0, 0x44, 0x80, 0xF0, 0xE5, 0x22, 0x54,
-/*0120*/0x30, 0x65, 0x08, 0x70, 0x09, 0x12, 0x1B, 0x83,
- 0xE0, 0x54, 0xBF, 0xF0, 0x80, 0x09, 0x12, 0x1B,
-/*0130*/0x83, 0x74, 0x40, 0xF0, 0x02, 0x04, 0x00, 0x12,
- 0x1B, 0xA2, 0x75, 0x83, 0xAE, 0x74, 0xFF, 0xF0,
-/*0140*/0xAF, 0x08, 0x7E, 0x00, 0xEF, 0x44, 0x07, 0xF5,
- 0x82, 0xE0, 0xFD, 0xE5, 0x0B, 0x25, 0xE0, 0x25,
-/*0150*/0xE0, 0x24, 0x81, 0xF5, 0x82, 0xE4, 0x34, 0x07,
- 0xF5, 0x83, 0xED, 0xF0, 0x90, 0x07, 0x0E, 0xE0,
-/*0160*/0x04, 0xF0, 0xEF, 0x44, 0x07, 0xF5, 0x82, 0x75,
- 0x83, 0x98, 0xE0, 0xF5, 0x28, 0x12, 0x1B, 0xB3,
-/*0170*/0x40, 0x0C, 0x12, 0x1B, 0x83, 0xE0, 0x44, 0x01,
- 0x12, 0x1B, 0xC2, 0x02, 0x03, 0xFB, 0xAF, 0x08,
-/*0180*/0x7E, 0x00, 0x74, 0x80, 0xCD, 0xEF, 0xCD, 0x8D,
- 0x82, 0xF5, 0x83, 0xE0, 0x30, 0xE0, 0x0A, 0x12,
-/*0190*/0x1B, 0x83, 0xE0, 0x44, 0x20, 0xF0, 0x02, 0x04,
- 0x00, 0x12, 0x1B, 0x83, 0xE0, 0x54, 0xDF, 0xF0,
-/*01A0*/0xEE, 0x44, 0xAE, 0x12, 0x1B, 0xCA, 0x30, 0xE4,
- 0x03, 0x02, 0x04, 0x00, 0x74, 0x9E, 0x12, 0x1B,
-/*01B0*/0x95, 0x20, 0xE0, 0x03, 0x02, 0x04, 0x00, 0x8F,
- 0x82, 0x8E, 0x83, 0xE0, 0x20, 0xE0, 0x03, 0x02,
-/*01C0*/0x04, 0x00, 0x12, 0x1B, 0x83, 0xE0, 0x44, 0x10,
- 0xF0, 0xE5, 0x08, 0x44, 0x06, 0xF5, 0x82, 0x75,
-/*01D0*/0x83, 0x9E, 0xE0, 0x44, 0x04, 0xF0, 0xAF, 0x08,
- 0x7E, 0x00, 0xEF, 0x44, 0x06, 0xF5, 0x82, 0xE0,
-/*01E0*/0x20, 0xE2, 0x34, 0x12, 0x1B, 0x83, 0xE0, 0x44,
- 0x08, 0xF0, 0xE5, 0xE4, 0x30, 0xE6, 0x04, 0x7D,
-/*01F0*/0x01, 0x80, 0x02, 0x7D, 0x00, 0xE5, 0x7E, 0xC3,
- 0x94, 0x04, 0x50, 0x04, 0x7C, 0x01, 0x80, 0x02,
-/*0200*/0x7C, 0x00, 0xEC, 0x4D, 0x60, 0x05, 0xC2, 0x35,
- 0x02, 0x04, 0x00, 0xEE, 0x44, 0xD2, 0x12, 0x1B,
-/*0210*/0xCA, 0x44, 0x40, 0xF0, 0x02, 0x04, 0x00, 0x12,
- 0x1B, 0x83, 0xE0, 0x54, 0xF7, 0xF0, 0x12, 0x1B,
-/*0220*/0xA2, 0x75, 0x83, 0xD2, 0xE0, 0x54, 0xBF, 0xF0,
- 0x90, 0x07, 0x14, 0xE0, 0x04, 0xF0, 0xE5, 0x7E,
-/*0230*/0x70, 0x03, 0x75, 0x7E, 0x01, 0xAF, 0x08, 0x7E,
- 0x00, 0x12, 0x1B, 0xB3, 0x40, 0x12, 0x12, 0x1B,
-/*0240*/0x83, 0xE0, 0x44, 0x01, 0x12, 0x1B, 0x82, 0xE0,
- 0x54, 0x02, 0x12, 0x1B, 0xC2, 0x02, 0x04, 0x00,
-/*0250*/0x12, 0x1B, 0x83, 0xE0, 0x44, 0x02, 0x12, 0x1B,
- 0x82, 0xE0, 0x54, 0xFE, 0xF0, 0xC2, 0x35, 0xEE,
-/*0260*/0x44, 0x8A, 0x8F, 0x82, 0xF5, 0x83, 0xE0, 0xF5,
- 0x17, 0x54, 0x8F, 0x44, 0x40, 0xF0, 0x74, 0x90,
-/*0270*/0xFC, 0xE5, 0x08, 0x44, 0x07, 0xFD, 0xF5, 0x82,
- 0x8C, 0x83, 0xE0, 0x54, 0x3F, 0x90, 0x07, 0x02,
-/*0280*/0xF0, 0xE0, 0x54, 0xC0, 0x8D, 0x82, 0x8C, 0x83,
- 0xF0, 0x74, 0x92, 0x12, 0x1B, 0x95, 0x90, 0x07,
-/*0290*/0x03, 0x12, 0x1B, 0xA9, 0x74, 0x82, 0x12, 0x1B,
- 0x95, 0x90, 0x07, 0x04, 0x12, 0x1B, 0xA9, 0x74,
-/*02A0*/0xB4, 0x12, 0x1B, 0x95, 0x90, 0x07, 0x05, 0x12,
- 0x1B, 0xA9, 0x74, 0x94, 0xFE, 0xE5, 0x08, 0x44,
-/*02B0*/0x06, 0x12, 0x1B, 0x9A, 0xF5, 0x10, 0x30, 0xE0,
- 0x04, 0xD2, 0x37, 0x80, 0x02, 0xC2, 0x37, 0xE5,
-/*02C0*/0x10, 0x54, 0x7F, 0x8F, 0x82, 0x8E, 0x83, 0xF0,
- 0x30, 0x44, 0x30, 0x12, 0x1B, 0x93, 0x54, 0x80,
-/*02D0*/0xD3, 0x94, 0x00, 0x40, 0x04, 0xD2, 0x39, 0x80,
- 0x02, 0xC2, 0x39, 0x8F, 0x82, 0x8E, 0x83, 0xE0,
-/*02E0*/0x44, 0x80, 0xF0, 0x12, 0x1B, 0x93, 0x54, 0x40,
- 0xD3, 0x94, 0x00, 0x40, 0x04, 0xD2, 0x3A, 0x80,
-/*02F0*/0x02, 0xC2, 0x3A, 0x8F, 0x82, 0x8E, 0x83, 0xE0,
- 0x44, 0x40, 0xF0, 0x74, 0x92, 0xFE, 0xE5, 0x08,
-/*0300*/0x44, 0x06, 0x12, 0x1B, 0x9A, 0x30, 0xE7, 0x04,
- 0xD2, 0x38, 0x80, 0x02, 0xC2, 0x38, 0x8F, 0x82,
-/*0310*/0x8E, 0x83, 0xE0, 0x54, 0x7F, 0xF0, 0x12, 0x1E,
- 0x17, 0xE4, 0xF5, 0x0A, 0x20, 0x03, 0x02, 0x80,
-/*0320*/0x03, 0x30, 0x43, 0x03, 0x12, 0x19, 0x17, 0x20,
- 0x02, 0x02, 0x80, 0x03, 0x30, 0x42, 0x03, 0x12,
-/*0330*/0x0C, 0x97, 0x30, 0x30, 0x06, 0x12, 0x19, 0x17,
- 0x12, 0x0C, 0x97, 0x12, 0x0D, 0x4F, 0x12, 0x1B,
-/*0340*/0x83, 0xE0, 0x54, 0xFB, 0xF0, 0xE5, 0x0A, 0xC3,
- 0x94, 0x01, 0x40, 0x46, 0x43, 0xE1, 0x08, 0x12,
-/*0350*/0x1B, 0x83, 0xE0, 0x44, 0x04, 0xF0, 0xE5, 0xE4,
- 0x20, 0xE7, 0x2A, 0x12, 0x1B, 0xA2, 0x75, 0x83,
-/*0360*/0xD2, 0xE0, 0x54, 0x08, 0xD3, 0x94, 0x00, 0x40,
- 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5,
-/*0370*/0x0A, 0xC3, 0x94, 0x01, 0x40, 0x04, 0x7E, 0x01,
- 0x80, 0x02, 0x7E, 0x00, 0xEF, 0x5E, 0x60, 0x05,
-/*0380*/0x12, 0x1D, 0xA8, 0x80, 0x17, 0x12, 0x1B, 0xA2,
- 0x75, 0x83, 0xD2, 0xE0, 0x44, 0x08, 0xF0, 0x02,
-/*0390*/0x04, 0x00, 0x12, 0x1B, 0xA2, 0x75, 0x83, 0xD2,
- 0xE0, 0x54, 0xF7, 0xF0, 0x12, 0x1E, 0x17, 0x7F,
-/*03A0*/0x08, 0x12, 0x16, 0xB3, 0x74, 0x8E, 0xFE, 0x12,
- 0x1B, 0xA2, 0x8E, 0x83, 0xE0, 0xF5, 0x10, 0x54,
-/*03B0*/0xFE, 0xF0, 0xE5, 0x10, 0x44, 0x01, 0xFF, 0xE5,
- 0x08, 0xFD, 0xED, 0x44, 0x07, 0xF5, 0x82, 0xEF,
-/*03C0*/0xF0, 0xE5, 0x10, 0x54, 0xFE, 0xFF, 0xED, 0x44,
- 0x07, 0xF5, 0x82, 0xEF, 0x12, 0x1B, 0xA1, 0x75,
-/*03D0*/0x83, 0x86, 0xE0, 0x44, 0x10, 0x12, 0x1B, 0xA1,
- 0xE0, 0x44, 0x10, 0xF0, 0x12, 0x1B, 0x83, 0xE0,
-/*03E0*/0x54, 0xFD, 0x44, 0x01, 0xFF, 0x12, 0x1B, 0x83,
- 0xEF, 0x12, 0x1B, 0xC2, 0x30, 0x32, 0x0C, 0xE5,
-/*03F0*/0x08, 0x44, 0x08, 0xF5, 0x82, 0x75, 0x83, 0x82,
- 0x74, 0x05, 0xF0, 0xAF, 0x0B, 0x12, 0x18, 0x59,
-/*0400*/0x74, 0x10, 0x25, 0x08, 0xF5, 0x08, 0x02, 0x00,
- 0x85, 0x05, 0x09, 0xE5, 0x09, 0xD3, 0x94, 0x07,
-/*0410*/0x50, 0x03, 0x02, 0x00, 0x82, 0xE5, 0x7E, 0xD3,
- 0x94, 0x00, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02,
-/*0420*/0x7F, 0x00, 0xE5, 0x7E, 0xC3, 0x94, 0xFA, 0x50,
- 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE,
-/*0430*/0x5F, 0x60, 0x02, 0x05, 0x7E, 0x30, 0x35, 0x0B,
- 0x43, 0xE1, 0x01, 0x7F, 0x09, 0x12, 0x16, 0xB3,
-/*0440*/0x02, 0x00, 0x58, 0x53, 0xE1, 0xFE, 0x02, 0x00,
- 0x58, 0x8E, 0x6A, 0x8F, 0x6B, 0x8C, 0x6C, 0x8D,
-/*0450*/0x6D, 0x75, 0x6E, 0x01, 0x75, 0x6F, 0x01, 0x75,
- 0x70, 0x01, 0xE4, 0xF5, 0x73, 0xF5, 0x74, 0xF5,
-/*0460*/0x75, 0x90, 0x07, 0x2F, 0xF0, 0xF5, 0x3C, 0xF5,
- 0x3E, 0xF5, 0x46, 0xF5, 0x47, 0xF5, 0x3D, 0xF5,
-/*0470*/0x3F, 0xF5, 0x6F, 0xE5, 0x6F, 0x70, 0x0E, 0xE5,
- 0x6B, 0x45, 0x6A, 0x12, 0x07, 0x37, 0x75, 0x83,
-/*0480*/0x80, 0x74, 0x3A, 0x80, 0x08, 0x12, 0x07, 0x37,
- 0x75, 0x83, 0x80, 0x74, 0x1A, 0x12, 0x07, 0x36,
-/*0490*/0x75, 0x83, 0x86, 0xE0, 0x44, 0x08, 0xF0, 0xE4,
- 0xF5, 0x6E, 0xC3, 0x74, 0x3F, 0x95, 0x6E, 0xFF,
-/*04A0*/0x12, 0x08, 0x68, 0x75, 0x83, 0x82, 0xEF, 0xF0,
- 0x12, 0x19, 0x74, 0x12, 0x08, 0xD3, 0xE5, 0x33,
-/*04B0*/0xF0, 0x12, 0x09, 0x07, 0x12, 0x08, 0xBE, 0x40,
- 0xE1, 0xE5, 0x6F, 0x70, 0x0B, 0x12, 0x07, 0x37,
-/*04C0*/0x75, 0x83, 0x80, 0x74, 0x36, 0xF0, 0x80, 0x09,
- 0x12, 0x07, 0x37, 0x75, 0x83, 0x80, 0x74, 0x16,
-/*04D0*/0xF0, 0x75, 0x6E, 0x01, 0x12, 0x07, 0x37, 0x75,
- 0x83, 0xB4, 0xE5, 0x6E, 0xF0, 0x12, 0x19, 0x74,
-/*04E0*/0x74, 0x3F, 0x25, 0x6E, 0xF5, 0x82, 0xE4, 0x34,
- 0x00, 0xF5, 0x83, 0xE5, 0x33, 0xF0, 0x74, 0xBF,
-/*04F0*/0x25, 0x6E, 0xF5, 0x82, 0xE4, 0x34, 0x00, 0x12,
- 0x08, 0xBE, 0x40, 0xD8, 0xE4, 0xF5, 0x70, 0xF5,
-/*0500*/0x46, 0xF5, 0x47, 0xF5, 0x6E, 0x12, 0x09, 0x07,
- 0xF5, 0x83, 0xE0, 0xFE, 0x12, 0x08, 0xD3, 0xE0,
-/*0510*/0x7C, 0x00, 0x24, 0x00, 0xFF, 0xEC, 0x3E, 0xFE,
- 0xAD, 0x3B, 0xD3, 0xEF, 0x9D, 0xEE, 0x9C, 0x50,
-/*0520*/0x04, 0x7B, 0x01, 0x80, 0x02, 0x7B, 0x00, 0xE5,
- 0x70, 0x70, 0x04, 0x7A, 0x01, 0x80, 0x02, 0x7A,
-/*0530*/0x00, 0xEB, 0x5A, 0x60, 0x06, 0x85, 0x6E, 0x46,
- 0x75, 0x70, 0x01, 0xD3, 0xEF, 0x9D, 0xEE, 0x9C,
-/*0540*/0x50, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00,
- 0xE5, 0x70, 0xB4, 0x01, 0x04, 0x7E, 0x01, 0x80,
-/*0550*/0x02, 0x7E, 0x00, 0xEF, 0x5E, 0x60, 0x03, 0x85,
- 0x6E, 0x47, 0x05, 0x6E, 0xE5, 0x6E, 0x64, 0x7F,
-/*0560*/0x70, 0xA3, 0xE5, 0x46, 0x60, 0x05, 0xE5, 0x47,
- 0xB4, 0x7E, 0x03, 0x85, 0x46, 0x47, 0xE5, 0x6F,
-/*0570*/0x70, 0x08, 0x85, 0x46, 0x76, 0x85, 0x47, 0x77,
- 0x80, 0x0E, 0xC3, 0x74, 0x7F, 0x95, 0x46, 0xF5,
-/*0580*/0x78, 0xC3, 0x74, 0x7F, 0x95, 0x47, 0xF5, 0x79,
- 0xE5, 0x6F, 0x70, 0x37, 0xE5, 0x46, 0x65, 0x47,
-/*0590*/0x70, 0x0C, 0x75, 0x73, 0x01, 0x75, 0x74, 0x01,
- 0xF5, 0x3C, 0xF5, 0x3D, 0x80, 0x35, 0xE4, 0xF5,
-/*05A0*/0x4E, 0xC3, 0xE5, 0x47, 0x95, 0x46, 0xF5, 0x3C,
- 0xC3, 0x13, 0xF5, 0x71, 0x25, 0x46, 0xF5, 0x72,
-/*05B0*/0xC3, 0x94, 0x3F, 0x40, 0x05, 0xE4, 0xF5, 0x3D,
- 0x80, 0x40, 0xC3, 0x74, 0x3F, 0x95, 0x72, 0xF5,
-/*05C0*/0x3D, 0x80, 0x37, 0xE5, 0x46, 0x65, 0x47, 0x70,
- 0x0F, 0x75, 0x73, 0x01, 0x75, 0x75, 0x01, 0xF5,
-/*05D0*/0x3E, 0xF5, 0x3F, 0x75, 0x4E, 0x01, 0x80, 0x22,
- 0xE4, 0xF5, 0x4E, 0xC3, 0xE5, 0x47, 0x95, 0x46,
-/*05E0*/0xF5, 0x3E, 0xC3, 0x13, 0xF5, 0x71, 0x25, 0x46,
- 0xF5, 0x72, 0xD3, 0x94, 0x3F, 0x50, 0x05, 0xE4,
-/*05F0*/0xF5, 0x3F, 0x80, 0x06, 0xE5, 0x72, 0x24, 0xC1,
- 0xF5, 0x3F, 0x05, 0x6F, 0xE5, 0x6F, 0xC3, 0x94,
-/*0600*/0x02, 0x50, 0x03, 0x02, 0x04, 0x73, 0xE5, 0x6D,
- 0x45, 0x6C, 0x70, 0x02, 0x80, 0x04, 0xE5, 0x74,
-/*0610*/0x45, 0x75, 0x90, 0x07, 0x2F, 0xF0, 0x7F, 0x01,
- 0xE5, 0x3E, 0x60, 0x04, 0xE5, 0x3C, 0x70, 0x14,
-/*0620*/0xE4, 0xF5, 0x3C, 0xF5, 0x3D, 0xF5, 0x3E, 0xF5,
- 0x3F, 0x12, 0x08, 0xDF, 0x70, 0x04, 0xF0, 0x02,
-/*0630*/0x06, 0xB1, 0x80, 0x7A, 0xE5, 0x3C, 0xC3, 0x95,
- 0x3E, 0x40, 0x07, 0xE5, 0x3C, 0x95, 0x3E, 0xFF,
-/*0640*/0x80, 0x06, 0xC3, 0xE5, 0x3E, 0x95, 0x3C, 0xFF,
- 0xE5, 0x76, 0xD3, 0x95, 0x79, 0x40, 0x05, 0x85,
-/*0650*/0x76, 0x7A, 0x80, 0x03, 0x85, 0x79, 0x7A, 0xE5,
- 0x77, 0xC3, 0x95, 0x78, 0x50, 0x05, 0x85, 0x77,
-/*0660*/0x7B, 0x80, 0x03, 0x85, 0x78, 0x7B, 0xE5, 0x7B,
- 0xD3, 0x95, 0x7A, 0x40, 0x30, 0xE5, 0x7B, 0x95,
-/*0670*/0x7A, 0xF5, 0x3C, 0xF5, 0x3E, 0xC3, 0xE5, 0x7B,
- 0x95, 0x7A, 0x90, 0x07, 0x19, 0xF0, 0xE5, 0x3C,
-/*0680*/0xC3, 0x13, 0xF5, 0x71, 0x25, 0x7A, 0xF5, 0x72,
- 0xC3, 0x94, 0x3F, 0x40, 0x05, 0xE4, 0xF5, 0x3D,
-/*0690*/0x80, 0x1F, 0xC3, 0x74, 0x3F, 0x95, 0x72, 0xF5,
- 0x3D, 0xF5, 0x3F, 0x80, 0x14, 0xE4, 0xF5, 0x3C,
-/*06A0*/0xF5, 0x3E, 0x90, 0x07, 0x19, 0xF0, 0x12, 0x08,
- 0xDF, 0x70, 0x03, 0xF0, 0x80, 0x03, 0x74, 0x01,
-/*06B0*/0xF0, 0x12, 0x08, 0x68, 0x75, 0x83, 0xD0, 0xE0,
- 0x54, 0x0F, 0xFE, 0xAD, 0x3C, 0x70, 0x02, 0x7E,
-/*06C0*/0x07, 0xBE, 0x0F, 0x02, 0x7E, 0x80, 0xEE, 0xFB,
- 0xEF, 0xD3, 0x9B, 0x74, 0x80, 0xF8, 0x98, 0x40,
-/*06D0*/0x1F, 0xE4, 0xF5, 0x3C, 0xF5, 0x3E, 0x12, 0x08,
- 0xDF, 0x70, 0x03, 0xF0, 0x80, 0x12, 0x74, 0x01,
-/*06E0*/0xF0, 0xE5, 0x08, 0xFB, 0xEB, 0x44, 0x07, 0xF5,
- 0x82, 0x75, 0x83, 0xD2, 0xE0, 0x44, 0x10, 0xF0,
-/*06F0*/0xE5, 0x08, 0xFB, 0xEB, 0x44, 0x09, 0xF5, 0x82,
- 0x75, 0x83, 0x9E, 0xED, 0xF0, 0xEB, 0x44, 0x07,
-/*0700*/0xF5, 0x82, 0x75, 0x83, 0xCA, 0xED, 0xF0, 0x12,
- 0x08, 0x68, 0x75, 0x83, 0xCC, 0xEF, 0xF0, 0x22,
+/*0100*/0x07, 0x12, 0x1A, 0x3C, 0xF5, 0x22, 0x54, 0x40,
+ 0xD3, 0x94, 0x00, 0x40, 0x1E, 0xE5, 0x29, 0x54,
+/*0110*/0xF0, 0x70, 0x21, 0x12, 0x19, 0xF3, 0xE0, 0x44,
+ 0x80, 0xF0, 0xE5, 0x22, 0x54, 0x30, 0x65, 0x08,
+/*0120*/0x70, 0x09, 0x12, 0x19, 0xF3, 0xE0, 0x54, 0xBF,
+ 0xF0, 0x80, 0x09, 0x12, 0x19, 0xF3, 0x74, 0x40,
+/*0130*/0xF0, 0x02, 0x03, 0xFB, 0x12, 0x1A, 0x12, 0x75,
+ 0x83, 0xAE, 0x74, 0xFF, 0xF0, 0xAF, 0x08, 0x7E,
+/*0140*/0x00, 0xEF, 0x44, 0x07, 0xF5, 0x82, 0xE0, 0xFD,
+ 0xE5, 0x0B, 0x25, 0xE0, 0x25, 0xE0, 0x24, 0x81,
+/*0150*/0xF5, 0x82, 0xE4, 0x34, 0x07, 0xF5, 0x83, 0xED,
+ 0xF0, 0x90, 0x07, 0x0E, 0xE0, 0x04, 0xF0, 0xEF,
+/*0160*/0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0x98, 0xE0,
+ 0xF5, 0x28, 0x12, 0x1A, 0x23, 0x40, 0x0C, 0x12,
+/*0170*/0x19, 0xF3, 0xE0, 0x44, 0x01, 0x12, 0x1A, 0x32,
+ 0x02, 0x03, 0xF6, 0xAF, 0x08, 0x7E, 0x00, 0x74,
+/*0180*/0x80, 0xCD, 0xEF, 0xCD, 0x8D, 0x82, 0xF5, 0x83,
+ 0xE0, 0x30, 0xE0, 0x0A, 0x12, 0x19, 0xF3, 0xE0,
+/*0190*/0x44, 0x20, 0xF0, 0x02, 0x03, 0xFB, 0x12, 0x19,
+ 0xF3, 0xE0, 0x54, 0xDF, 0xF0, 0xEE, 0x44, 0xAE,
+/*01A0*/0x12, 0x1A, 0x43, 0x30, 0xE4, 0x03, 0x02, 0x03,
+ 0xFB, 0x74, 0x9E, 0x12, 0x1A, 0x05, 0x20, 0xE0,
+/*01B0*/0x03, 0x02, 0x03, 0xFB, 0x8F, 0x82, 0x8E, 0x83,
+ 0xE0, 0x20, 0xE0, 0x03, 0x02, 0x03, 0xFB, 0x12,
+/*01C0*/0x19, 0xF3, 0xE0, 0x44, 0x10, 0xF0, 0xE5, 0xE3,
+ 0x20, 0xE7, 0x08, 0xE5, 0x08, 0x12, 0x1A, 0x3A,
+/*01D0*/0x44, 0x04, 0xF0, 0xAF, 0x08, 0x7E, 0x00, 0xEF,
+ 0x12, 0x1A, 0x3A, 0x20, 0xE2, 0x34, 0x12, 0x19,
+/*01E0*/0xF3, 0xE0, 0x44, 0x08, 0xF0, 0xE5, 0xE4, 0x30,
+ 0xE6, 0x04, 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00,
+/*01F0*/0xE5, 0x7E, 0xC3, 0x94, 0x04, 0x50, 0x04, 0x7C,
+ 0x01, 0x80, 0x02, 0x7C, 0x00, 0xEC, 0x4D, 0x60,
+/*0200*/0x05, 0xC2, 0x35, 0x02, 0x03, 0xFB, 0xEE, 0x44,
+ 0xD2, 0x12, 0x1A, 0x43, 0x44, 0x40, 0xF0, 0x02,
+/*0210*/0x03, 0xFB, 0x12, 0x19, 0xF3, 0xE0, 0x54, 0xF7,
+ 0xF0, 0x12, 0x1A, 0x12, 0x75, 0x83, 0xD2, 0xE0,
+/*0220*/0x54, 0xBF, 0xF0, 0x90, 0x07, 0x14, 0xE0, 0x04,
+ 0xF0, 0xE5, 0x7E, 0x70, 0x03, 0x75, 0x7E, 0x01,
+/*0230*/0xAF, 0x08, 0x7E, 0x00, 0x12, 0x1A, 0x23, 0x40,
+ 0x12, 0x12, 0x19, 0xF3, 0xE0, 0x44, 0x01, 0x12,
+/*0240*/0x19, 0xF2, 0xE0, 0x54, 0x02, 0x12, 0x1A, 0x32,
+ 0x02, 0x03, 0xFB, 0x12, 0x19, 0xF3, 0xE0, 0x44,
+/*0250*/0x02, 0x12, 0x19, 0xF2, 0xE0, 0x54, 0xFE, 0xF0,
+ 0xC2, 0x35, 0xEE, 0x44, 0x8A, 0x8F, 0x82, 0xF5,
+/*0260*/0x83, 0xE0, 0xF5, 0x17, 0x54, 0x8F, 0x44, 0x40,
+ 0xF0, 0x74, 0x90, 0xFC, 0xE5, 0x08, 0x44, 0x07,
+/*0270*/0xFD, 0xF5, 0x82, 0x8C, 0x83, 0xE0, 0x54, 0x3F,
+ 0x90, 0x07, 0x02, 0xF0, 0xE0, 0x54, 0xC0, 0x8D,
+/*0280*/0x82, 0x8C, 0x83, 0xF0, 0x74, 0x92, 0x12, 0x1A,
+ 0x05, 0x90, 0x07, 0x03, 0x12, 0x1A, 0x19, 0x74,
+/*0290*/0x82, 0x12, 0x1A, 0x05, 0x90, 0x07, 0x04, 0x12,
+ 0x1A, 0x19, 0x74, 0xB4, 0x12, 0x1A, 0x05, 0x90,
+/*02A0*/0x07, 0x05, 0x12, 0x1A, 0x19, 0x74, 0x94, 0xFE,
+ 0xE5, 0x08, 0x44, 0x06, 0x12, 0x1A, 0x0A, 0xF5,
+/*02B0*/0x10, 0x30, 0xE0, 0x04, 0xD2, 0x37, 0x80, 0x02,
+ 0xC2, 0x37, 0xE5, 0x10, 0x54, 0x7F, 0x8F, 0x82,
+/*02C0*/0x8E, 0x83, 0xF0, 0x30, 0x44, 0x30, 0x12, 0x1A,
+ 0x03, 0x54, 0x80, 0xD3, 0x94, 0x00, 0x40, 0x04,
+/*02D0*/0xD2, 0x39, 0x80, 0x02, 0xC2, 0x39, 0x8F, 0x82,
+ 0x8E, 0x83, 0xE0, 0x44, 0x80, 0xF0, 0x12, 0x1A,
+/*02E0*/0x03, 0x54, 0x40, 0xD3, 0x94, 0x00, 0x40, 0x04,
+ 0xD2, 0x3A, 0x80, 0x02, 0xC2, 0x3A, 0x8F, 0x82,
+/*02F0*/0x8E, 0x83, 0xE0, 0x44, 0x40, 0xF0, 0x74, 0x92,
+ 0xFE, 0xE5, 0x08, 0x44, 0x06, 0x12, 0x1A, 0x0A,
+/*0300*/0x30, 0xE7, 0x04, 0xD2, 0x38, 0x80, 0x02, 0xC2,
+ 0x38, 0x8F, 0x82, 0x8E, 0x83, 0xE0, 0x54, 0x7F,
+/*0310*/0xF0, 0x12, 0x1E, 0x46, 0xE4, 0xF5, 0x0A, 0x20,
+ 0x03, 0x02, 0x80, 0x03, 0x30, 0x43, 0x03, 0x12,
+/*0320*/0x19, 0x95, 0x20, 0x02, 0x02, 0x80, 0x03, 0x30,
+ 0x42, 0x03, 0x12, 0x0C, 0x8F, 0x30, 0x30, 0x06,
+/*0330*/0x12, 0x19, 0x95, 0x12, 0x0C, 0x8F, 0x12, 0x0D,
+ 0x47, 0x12, 0x19, 0xF3, 0xE0, 0x54, 0xFB, 0xF0,
+/*0340*/0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x40, 0x46, 0x43,
+ 0xE1, 0x08, 0x12, 0x19, 0xF3, 0xE0, 0x44, 0x04,
+/*0350*/0xF0, 0xE5, 0xE4, 0x20, 0xE7, 0x2A, 0x12, 0x1A,
+ 0x12, 0x75, 0x83, 0xD2, 0xE0, 0x54, 0x08, 0xD3,
+/*0360*/0x94, 0x00, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02,
+ 0x7F, 0x00, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x40,
+/*0370*/0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEF,
+ 0x5E, 0x60, 0x05, 0x12, 0x1D, 0xD7, 0x80, 0x17,
+/*0380*/0x12, 0x1A, 0x12, 0x75, 0x83, 0xD2, 0xE0, 0x44,
+ 0x08, 0xF0, 0x02, 0x03, 0xFB, 0x12, 0x1A, 0x12,
+/*0390*/0x75, 0x83, 0xD2, 0xE0, 0x54, 0xF7, 0xF0, 0x12,
+ 0x1E, 0x46, 0x7F, 0x08, 0x12, 0x17, 0x31, 0x74,
+/*03A0*/0x8E, 0xFE, 0x12, 0x1A, 0x12, 0x8E, 0x83, 0xE0,
+ 0xF5, 0x10, 0x54, 0xFE, 0xF0, 0xE5, 0x10, 0x44,
+/*03B0*/0x01, 0xFF, 0xE5, 0x08, 0xFD, 0xED, 0x44, 0x07,
+ 0xF5, 0x82, 0xEF, 0xF0, 0xE5, 0x10, 0x54, 0xFE,
+/*03C0*/0xFF, 0xED, 0x44, 0x07, 0xF5, 0x82, 0xEF, 0x12,
+ 0x1A, 0x11, 0x75, 0x83, 0x86, 0xE0, 0x44, 0x10,
+/*03D0*/0x12, 0x1A, 0x11, 0xE0, 0x44, 0x10, 0xF0, 0x12,
+ 0x19, 0xF3, 0xE0, 0x54, 0xFD, 0x44, 0x01, 0xFF,
+/*03E0*/0x12, 0x19, 0xF3, 0xEF, 0x12, 0x1A, 0x32, 0x30,
+ 0x32, 0x0C, 0xE5, 0x08, 0x44, 0x08, 0xF5, 0x82,
+/*03F0*/0x75, 0x83, 0x82, 0x74, 0x05, 0xF0, 0xAF, 0x0B,
+ 0x12, 0x18, 0xD7, 0x74, 0x10, 0x25, 0x08, 0xF5,
+/*0400*/0x08, 0x02, 0x00, 0x85, 0x05, 0x09, 0xE5, 0x09,
+ 0xD3, 0x94, 0x07, 0x50, 0x03, 0x02, 0x00, 0x82,
+/*0410*/0xE5, 0x7E, 0xD3, 0x94, 0x00, 0x40, 0x04, 0x7F,
+ 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x7E, 0xC3,
+/*0420*/0x94, 0xFA, 0x50, 0x04, 0x7E, 0x01, 0x80, 0x02,
+ 0x7E, 0x00, 0xEE, 0x5F, 0x60, 0x02, 0x05, 0x7E,
+/*0430*/0x30, 0x35, 0x0B, 0x43, 0xE1, 0x01, 0x7F, 0x09,
+ 0x12, 0x17, 0x31, 0x02, 0x00, 0x58, 0x53, 0xE1,
+/*0440*/0xFE, 0x02, 0x00, 0x58, 0x8E, 0x6A, 0x8F, 0x6B,
+ 0x8C, 0x6C, 0x8D, 0x6D, 0x75, 0x6E, 0x01, 0x75,
+/*0450*/0x6F, 0x01, 0x75, 0x70, 0x01, 0xE4, 0xF5, 0x73,
+ 0xF5, 0x74, 0xF5, 0x75, 0x90, 0x07, 0x2F, 0xF0,
+/*0460*/0xF5, 0x3C, 0xF5, 0x3E, 0xF5, 0x46, 0xF5, 0x47,
+ 0xF5, 0x3D, 0xF5, 0x3F, 0xF5, 0x6F, 0xE5, 0x6F,
+/*0470*/0x70, 0x0F, 0xE5, 0x6B, 0x45, 0x6A, 0x12, 0x07,
+ 0x2A, 0x75, 0x83, 0x80, 0x74, 0x3A, 0xF0, 0x80,
+/*0480*/0x09, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x80, 0x74,
+ 0x1A, 0xF0, 0xE4, 0xF5, 0x6E, 0xC3, 0x74, 0x3F,
+/*0490*/0x95, 0x6E, 0xFF, 0x12, 0x08, 0x65, 0x75, 0x83,
+ 0x82, 0xEF, 0xF0, 0x12, 0x1A, 0x4D, 0x12, 0x08,
+/*04A0*/0xC6, 0xE5, 0x33, 0xF0, 0x12, 0x08, 0xFA, 0x12,
+ 0x08, 0xB1, 0x40, 0xE1, 0xE5, 0x6F, 0x70, 0x0B,
+/*04B0*/0x12, 0x07, 0x2A, 0x75, 0x83, 0x80, 0x74, 0x36,
+ 0xF0, 0x80, 0x09, 0x12, 0x07, 0x2A, 0x75, 0x83,
+/*04C0*/0x80, 0x74, 0x16, 0xF0, 0x75, 0x6E, 0x01, 0x12,
+ 0x07, 0x2A, 0x75, 0x83, 0xB4, 0xE5, 0x6E, 0xF0,
+/*04D0*/0x12, 0x1A, 0x4D, 0x74, 0x3F, 0x25, 0x6E, 0xF5,
+ 0x82, 0xE4, 0x34, 0x00, 0xF5, 0x83, 0xE5, 0x33,
+/*04E0*/0xF0, 0x74, 0xBF, 0x25, 0x6E, 0xF5, 0x82, 0xE4,
+ 0x34, 0x00, 0x12, 0x08, 0xB1, 0x40, 0xD8, 0xE4,
+/*04F0*/0xF5, 0x70, 0xF5, 0x46, 0xF5, 0x47, 0xF5, 0x6E,
+ 0x12, 0x08, 0xFA, 0xF5, 0x83, 0xE0, 0xFE, 0x12,
+/*0500*/0x08, 0xC6, 0xE0, 0x7C, 0x00, 0x24, 0x00, 0xFF,
+ 0xEC, 0x3E, 0xFE, 0xAD, 0x3B, 0xD3, 0xEF, 0x9D,
+/*0510*/0xEE, 0x9C, 0x50, 0x04, 0x7B, 0x01, 0x80, 0x02,
+ 0x7B, 0x00, 0xE5, 0x70, 0x70, 0x04, 0x7A, 0x01,
+/*0520*/0x80, 0x02, 0x7A, 0x00, 0xEB, 0x5A, 0x60, 0x06,
+ 0x85, 0x6E, 0x46, 0x75, 0x70, 0x01, 0xD3, 0xEF,
+/*0530*/0x9D, 0xEE, 0x9C, 0x50, 0x04, 0x7F, 0x01, 0x80,
+ 0x02, 0x7F, 0x00, 0xE5, 0x70, 0xB4, 0x01, 0x04,
+/*0540*/0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEF, 0x5E,
+ 0x60, 0x03, 0x85, 0x6E, 0x47, 0x05, 0x6E, 0xE5,
+/*0550*/0x6E, 0x64, 0x7F, 0x70, 0xA3, 0xE5, 0x46, 0x60,
+ 0x05, 0xE5, 0x47, 0xB4, 0x7E, 0x03, 0x85, 0x46,
+/*0560*/0x47, 0xE5, 0x6F, 0x70, 0x08, 0x85, 0x46, 0x76,
+ 0x85, 0x47, 0x77, 0x80, 0x0E, 0xC3, 0x74, 0x7F,
+/*0570*/0x95, 0x46, 0xF5, 0x78, 0xC3, 0x74, 0x7F, 0x95,
+ 0x47, 0xF5, 0x79, 0xE5, 0x6F, 0x70, 0x37, 0xE5,
+/*0580*/0x46, 0x65, 0x47, 0x70, 0x0C, 0x75, 0x73, 0x01,
+ 0x75, 0x74, 0x01, 0xF5, 0x3C, 0xF5, 0x3D, 0x80,
+/*0590*/0x35, 0xE4, 0xF5, 0x4E, 0xC3, 0xE5, 0x47, 0x95,
+ 0x46, 0xF5, 0x3C, 0xC3, 0x13, 0xF5, 0x71, 0x25,
+/*05A0*/0x46, 0xF5, 0x72, 0xC3, 0x94, 0x3F, 0x40, 0x05,
+ 0xE4, 0xF5, 0x3D, 0x80, 0x40, 0xC3, 0x74, 0x3F,
+/*05B0*/0x95, 0x72, 0xF5, 0x3D, 0x80, 0x37, 0xE5, 0x46,
+ 0x65, 0x47, 0x70, 0x0F, 0x75, 0x73, 0x01, 0x75,
+/*05C0*/0x75, 0x01, 0xF5, 0x3E, 0xF5, 0x3F, 0x75, 0x4E,
+ 0x01, 0x80, 0x22, 0xE4, 0xF5, 0x4E, 0xC3, 0xE5,
+/*05D0*/0x47, 0x95, 0x46, 0xF5, 0x3E, 0xC3, 0x13, 0xF5,
+ 0x71, 0x25, 0x46, 0xF5, 0x72, 0xD3, 0x94, 0x3F,
+/*05E0*/0x50, 0x05, 0xE4, 0xF5, 0x3F, 0x80, 0x06, 0xE5,
+ 0x72, 0x24, 0xC1, 0xF5, 0x3F, 0x05, 0x6F, 0xE5,
+/*05F0*/0x6F, 0xC3, 0x94, 0x02, 0x50, 0x03, 0x02, 0x04,
+ 0x6E, 0xE5, 0x6D, 0x45, 0x6C, 0x70, 0x02, 0x80,
+/*0600*/0x04, 0xE5, 0x74, 0x45, 0x75, 0x90, 0x07, 0x2F,
+ 0xF0, 0x7F, 0x01, 0xE5, 0x3E, 0x60, 0x04, 0xE5,
+/*0610*/0x3C, 0x70, 0x14, 0xE4, 0xF5, 0x3C, 0xF5, 0x3D,
+ 0xF5, 0x3E, 0xF5, 0x3F, 0x12, 0x08, 0xD2, 0x70,
+/*0620*/0x04, 0xF0, 0x02, 0x06, 0xA4, 0x80, 0x7A, 0xE5,
+ 0x3C, 0xC3, 0x95, 0x3E, 0x40, 0x07, 0xE5, 0x3C,
+/*0630*/0x95, 0x3E, 0xFF, 0x80, 0x06, 0xC3, 0xE5, 0x3E,
+ 0x95, 0x3C, 0xFF, 0xE5, 0x76, 0xD3, 0x95, 0x79,
+/*0640*/0x40, 0x05, 0x85, 0x76, 0x7A, 0x80, 0x03, 0x85,
+ 0x79, 0x7A, 0xE5, 0x77, 0xC3, 0x95, 0x78, 0x50,
+/*0650*/0x05, 0x85, 0x77, 0x7B, 0x80, 0x03, 0x85, 0x78,
+ 0x7B, 0xE5, 0x7B, 0xD3, 0x95, 0x7A, 0x40, 0x30,
+/*0660*/0xE5, 0x7B, 0x95, 0x7A, 0xF5, 0x3C, 0xF5, 0x3E,
+ 0xC3, 0xE5, 0x7B, 0x95, 0x7A, 0x90, 0x07, 0x19,
+/*0670*/0xF0, 0xE5, 0x3C, 0xC3, 0x13, 0xF5, 0x71, 0x25,
+ 0x7A, 0xF5, 0x72, 0xC3, 0x94, 0x3F, 0x40, 0x05,
+/*0680*/0xE4, 0xF5, 0x3D, 0x80, 0x1F, 0xC3, 0x74, 0x3F,
+ 0x95, 0x72, 0xF5, 0x3D, 0xF5, 0x3F, 0x80, 0x14,
+/*0690*/0xE4, 0xF5, 0x3C, 0xF5, 0x3E, 0x90, 0x07, 0x19,
+ 0xF0, 0x12, 0x08, 0xD2, 0x70, 0x03, 0xF0, 0x80,
+/*06A0*/0x03, 0x74, 0x01, 0xF0, 0x12, 0x08, 0x65, 0x75,
+ 0x83, 0xD0, 0xE0, 0x54, 0x0F, 0xFE, 0xAD, 0x3C,
+/*06B0*/0x70, 0x02, 0x7E, 0x07, 0xBE, 0x0F, 0x02, 0x7E,
+ 0x80, 0xEE, 0xFB, 0xEF, 0xD3, 0x9B, 0x74, 0x80,
+/*06C0*/0xF8, 0x98, 0x40, 0x1F, 0xE4, 0xF5, 0x3C, 0xF5,
+ 0x3E, 0x12, 0x08, 0xD2, 0x70, 0x03, 0xF0, 0x80,
+/*06D0*/0x12, 0x74, 0x01, 0xF0, 0xE5, 0x08, 0xFB, 0xEB,
+ 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0xD2, 0xE0,
+/*06E0*/0x44, 0x10, 0xF0, 0xE5, 0x08, 0xFB, 0xEB, 0x44,
+ 0x09, 0xF5, 0x82, 0x75, 0x83, 0x9E, 0xED, 0xF0,
+/*06F0*/0xEB, 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0xCA,
+ 0xED, 0xF0, 0x12, 0x08, 0x65, 0x75, 0x83, 0xCC,
+/*0700*/0xEF, 0xF0, 0x22, 0xE5, 0x08, 0x44, 0x07, 0xF5,
+ 0x82, 0x75, 0x83, 0xBC, 0xE0, 0x54, 0xF0, 0xF0,
/*0710*/0xE5, 0x08, 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83,
- 0xBC, 0xE0, 0x54, 0xF0, 0xF0, 0xE5, 0x08, 0x44,
-/*0720*/0x07, 0xF5, 0x82, 0x75, 0x83, 0xBE, 0xE0, 0x54,
+ 0xBE, 0xE0, 0x54, 0xF0, 0xF0, 0xE5, 0x08, 0x44,
+/*0720*/0x07, 0xF5, 0x82, 0x75, 0x83, 0xC0, 0xE0, 0x54,
0xF0, 0xF0, 0xE5, 0x08, 0x44, 0x07, 0xF5, 0x82,
-/*0730*/0x75, 0x83, 0xC0, 0xE0, 0x54, 0xF0, 0xF0, 0xE5,
- 0x08, 0x44, 0x07, 0xF5, 0x82, 0x22, 0xF0, 0x90,
-/*0740*/0x07, 0x28, 0xE0, 0xFE, 0xA3, 0xE0, 0xF5, 0x82,
- 0x8E, 0x83, 0x22, 0x85, 0x42, 0x42, 0x85, 0x41,
-/*0750*/0x41, 0x85, 0x40, 0x40, 0x74, 0xC0, 0x2F, 0xF5,
- 0x82, 0x74, 0x02, 0x3E, 0xF5, 0x83, 0xE5, 0x42,
-/*0760*/0xF0, 0x74, 0xE0, 0x2F, 0xF5, 0x82, 0x74, 0x02,
- 0x3E, 0xF5, 0x83, 0x22, 0xE5, 0x42, 0x29, 0xFD,
-/*0770*/0xE4, 0x33, 0xFC, 0xE5, 0x3C, 0xC3, 0x9D, 0xEC,
- 0x64, 0x80, 0xF8, 0x74, 0x80, 0x98, 0x22, 0xF5,
-/*0780*/0x83, 0xE0, 0x90, 0x07, 0x22, 0x54, 0x1F, 0xFD,
- 0xE0, 0xFA, 0xA3, 0xE0, 0xF5, 0x82, 0x8A, 0x83,
-/*0790*/0xED, 0xF0, 0x22, 0x90, 0x07, 0x22, 0xE0, 0xFC,
- 0xA3, 0xE0, 0xF5, 0x82, 0x8C, 0x83, 0x22, 0x90,
-/*07A0*/0x07, 0x24, 0xFF, 0xED, 0x44, 0x07, 0xCF, 0xF0,
- 0xA3, 0xEF, 0xF0, 0x22, 0x85, 0x38, 0x38, 0x85,
-/*07B0*/0x39, 0x39, 0x85, 0x3A, 0x3A, 0x74, 0xC0, 0x2F,
- 0xF5, 0x82, 0x74, 0x02, 0x3E, 0xF5, 0x83, 0x22,
-/*07C0*/0x90, 0x07, 0x26, 0xFF, 0xED, 0x44, 0x07, 0xCF,
- 0xF0, 0xA3, 0xEF, 0xF0, 0x22, 0xF0, 0x74, 0xA0,
-/*07D0*/0x2F, 0xF5, 0x82, 0x74, 0x02, 0x3E, 0xF5, 0x83,
- 0x22, 0x74, 0xC0, 0x25, 0x11, 0xF5, 0x82, 0xE4,
-/*07E0*/0x34, 0x01, 0xF5, 0x83, 0x22, 0x74, 0x00, 0x25,
- 0x11, 0xF5, 0x82, 0xE4, 0x34, 0x02, 0xF5, 0x83,
-/*07F0*/0x22, 0x74, 0x60, 0x25, 0x11, 0xF5, 0x82, 0xE4,
- 0x34, 0x03, 0xF5, 0x83, 0x22, 0x74, 0x80, 0x25,
-/*0800*/0x11, 0xF5, 0x82, 0xE4, 0x34, 0x03, 0xF5, 0x83,
- 0x22, 0x74, 0xE0, 0x25, 0x11, 0xF5, 0x82, 0xE4,
-/*0810*/0x34, 0x03, 0xF5, 0x83, 0x22, 0x74, 0x40, 0x25,
- 0x11, 0xF5, 0x82, 0xE4, 0x34, 0x06, 0xF5, 0x83,
-/*0820*/0x22, 0x74, 0x80, 0x2F, 0xF5, 0x82, 0x74, 0x02,
- 0x3E, 0xF5, 0x83, 0x22, 0xF5, 0x83, 0xE5, 0x82,
-/*0830*/0x44, 0x07, 0xF5, 0x82, 0xE5, 0x40, 0xF0, 0x22,
- 0x74, 0x40, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34,
-/*0840*/0x02, 0xF5, 0x83, 0x22, 0x74, 0xC0, 0x25, 0x11,
+/*0730*/0x22, 0xF0, 0x90, 0x07, 0x28, 0xE0, 0xFE, 0xA3,
+ 0xE0, 0xF5, 0x82, 0x8E, 0x83, 0x22, 0x85, 0x42,
+/*0740*/0x42, 0x85, 0x41, 0x41, 0x85, 0x40, 0x40, 0x74,
+ 0xC0, 0x2F, 0xF5, 0x82, 0x74, 0x02, 0x3E, 0xF5,
+/*0750*/0x83, 0xE5, 0x42, 0xF0, 0x74, 0xE0, 0x2F, 0xF5,
+ 0x82, 0x74, 0x02, 0x3E, 0xF5, 0x83, 0x22, 0xE5,
+/*0760*/0x42, 0x29, 0xFD, 0xE4, 0x33, 0xFC, 0xE5, 0x3C,
+ 0xC3, 0x9D, 0xEC, 0x64, 0x80, 0xF8, 0x74, 0x80,
+/*0770*/0x98, 0x22, 0xF5, 0x83, 0xE0, 0x90, 0x07, 0x22,
+ 0x54, 0x1F, 0xFD, 0xE0, 0xFA, 0xA3, 0xE0, 0xF5,
+/*0780*/0x82, 0x8A, 0x83, 0xED, 0xF0, 0x22, 0x90, 0x07,
+ 0x22, 0xE0, 0xFC, 0xA3, 0xE0, 0xF5, 0x82, 0x8C,
+/*0790*/0x83, 0x22, 0x90, 0x07, 0x24, 0xFF, 0xED, 0x44,
+ 0x07, 0xCF, 0xF0, 0xA3, 0xEF, 0xF0, 0x22, 0x85,
+/*07A0*/0x38, 0x38, 0x85, 0x39, 0x39, 0x85, 0x3A, 0x3A,
+ 0x74, 0xC0, 0x2F, 0xF5, 0x82, 0x74, 0x02, 0x3E,
+/*07B0*/0xF5, 0x83, 0x22, 0x90, 0x07, 0x26, 0xFF, 0xED,
+ 0x44, 0x07, 0xCF, 0xF0, 0xA3, 0xEF, 0xF0, 0x22,
+/*07C0*/0xF0, 0x74, 0xA0, 0x2F, 0xF5, 0x82, 0x74, 0x02,
+ 0x3E, 0xF5, 0x83, 0x22, 0x74, 0xC0, 0x25, 0x11,
+/*07D0*/0xF5, 0x82, 0xE4, 0x34, 0x01, 0xF5, 0x83, 0x22,
+ 0x74, 0x00, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34,
+/*07E0*/0x02, 0xF5, 0x83, 0x22, 0x74, 0x60, 0x25, 0x11,
0xF5, 0x82, 0xE4, 0x34, 0x03, 0xF5, 0x83, 0x22,
-/*0850*/0x74, 0x00, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34,
- 0x06, 0xF5, 0x83, 0x22, 0x74, 0x20, 0x25, 0x11,
-/*0860*/0xF5, 0x82, 0xE4, 0x34, 0x06, 0xF5, 0x83, 0x22,
+/*07F0*/0x74, 0x80, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34,
+ 0x03, 0xF5, 0x83, 0x22, 0x74, 0xE0, 0x25, 0x11,
+/*0800*/0xF5, 0x82, 0xE4, 0x34, 0x03, 0xF5, 0x83, 0x22,
+ 0x74, 0x40, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34,
+/*0810*/0x06, 0xF5, 0x83, 0x22, 0x74, 0x80, 0x2F, 0xF5,
+ 0x82, 0x74, 0x02, 0x3E, 0xF5, 0x83, 0x22, 0xAF,
+/*0820*/0x08, 0x7E, 0x00, 0xEF, 0x44, 0x07, 0xF5, 0x82,
+ 0x22, 0xF5, 0x83, 0xE5, 0x82, 0x44, 0x07, 0xF5,
+/*0830*/0x82, 0xE5, 0x40, 0xF0, 0x22, 0x74, 0x40, 0x25,
+ 0x11, 0xF5, 0x82, 0xE4, 0x34, 0x02, 0xF5, 0x83,
+/*0840*/0x22, 0x74, 0xC0, 0x25, 0x11, 0xF5, 0x82, 0xE4,
+ 0x34, 0x03, 0xF5, 0x83, 0x22, 0x74, 0x00, 0x25,
+/*0850*/0x11, 0xF5, 0x82, 0xE4, 0x34, 0x06, 0xF5, 0x83,
+ 0x22, 0x74, 0x20, 0x25, 0x11, 0xF5, 0x82, 0xE4,
+/*0860*/0x34, 0x06, 0xF5, 0x83, 0x22, 0xE5, 0x08, 0xFD,
+ 0xED, 0x44, 0x07, 0xF5, 0x82, 0x22, 0xE5, 0x41,
+/*0870*/0xF0, 0xE5, 0x65, 0x64, 0x01, 0x45, 0x64, 0x22,
+ 0x7E, 0x00, 0xFB, 0x7A, 0x00, 0xFD, 0x7C, 0x00,
+/*0880*/0x22, 0x74, 0x20, 0x25, 0x11, 0xF5, 0x82, 0xE4,
+ 0x34, 0x02, 0x22, 0x74, 0xA0, 0x25, 0x11, 0xF5,
+/*0890*/0x82, 0xE4, 0x34, 0x03, 0x22, 0x85, 0x3E, 0x42,
+ 0x85, 0x3F, 0x41, 0x8F, 0x40, 0x22, 0x85, 0x3C,
+/*08A0*/0x42, 0x85, 0x3D, 0x41, 0x8F, 0x40, 0x22, 0x75,
+ 0x45, 0x3F, 0x90, 0x07, 0x20, 0xE4, 0xF0, 0xA3,
+/*08B0*/0x22, 0xF5, 0x83, 0xE5, 0x32, 0xF0, 0x05, 0x6E,
+ 0xE5, 0x6E, 0xC3, 0x94, 0x40, 0x22, 0xF0, 0xE5,
+/*08C0*/0x08, 0x44, 0x06, 0xF5, 0x82, 0x22, 0x74, 0x00,
+ 0x25, 0x6E, 0xF5, 0x82, 0xE4, 0x34, 0x00, 0xF5,
+/*08D0*/0x83, 0x22, 0xE5, 0x6D, 0x45, 0x6C, 0x90, 0x07,
+ 0x2F, 0x22, 0xE4, 0xF9, 0xE5, 0x3C, 0xD3, 0x95,
+/*08E0*/0x3E, 0x22, 0x74, 0x80, 0x2E, 0xF5, 0x82, 0xE4,
+ 0x34, 0x02, 0xF5, 0x83, 0xE0, 0x22, 0x74, 0xA0,
+/*08F0*/0x2E, 0xF5, 0x82, 0xE4, 0x34, 0x02, 0xF5, 0x83,
+ 0xE0, 0x22, 0x74, 0x80, 0x25, 0x6E, 0xF5, 0x82,
+/*0900*/0xE4, 0x34, 0x00, 0x22, 0x25, 0x42, 0xFD, 0xE4,
+ 0x33, 0xFC, 0x22, 0x85, 0x42, 0x42, 0x85, 0x41,
+/*0910*/0x41, 0x85, 0x40, 0x40, 0x22, 0xED, 0x4C, 0x60,
+ 0x03, 0x02, 0x09, 0xE5, 0xEF, 0x4E, 0x70, 0x37,
+/*0920*/0x90, 0x07, 0x26, 0x12, 0x07, 0x89, 0xE0, 0xFD,
+ 0x12, 0x07, 0xCC, 0xED, 0xF0, 0x90, 0x07, 0x28,
+/*0930*/0x12, 0x07, 0x89, 0xE0, 0xFD, 0x12, 0x07, 0xD8,
+ 0xED, 0xF0, 0x12, 0x07, 0x86, 0xE0, 0x54, 0x1F,
+/*0940*/0xFD, 0x12, 0x08, 0x81, 0xF5, 0x83, 0xED, 0xF0,
+ 0x90, 0x07, 0x24, 0x12, 0x07, 0x89, 0xE0, 0x54,
+/*0950*/0x1F, 0xFD, 0x12, 0x08, 0x35, 0xED, 0xF0, 0xEF,
+ 0x64, 0x04, 0x4E, 0x70, 0x37, 0x90, 0x07, 0x26,
+/*0960*/0x12, 0x07, 0x89, 0xE0, 0xFD, 0x12, 0x07, 0xE4,
+ 0xED, 0xF0, 0x90, 0x07, 0x28, 0x12, 0x07, 0x89,
+/*0970*/0xE0, 0xFD, 0x12, 0x07, 0xF0, 0xED, 0xF0, 0x12,
+ 0x07, 0x86, 0xE0, 0x54, 0x1F, 0xFD, 0x12, 0x08,
+/*0980*/0x8B, 0xF5, 0x83, 0xED, 0xF0, 0x90, 0x07, 0x24,
+ 0x12, 0x07, 0x89, 0xE0, 0x54, 0x1F, 0xFD, 0x12,
+/*0990*/0x08, 0x41, 0xED, 0xF0, 0xEF, 0x64, 0x01, 0x4E,
+ 0x70, 0x04, 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00,
+/*09A0*/0xEF, 0x64, 0x02, 0x4E, 0x70, 0x04, 0x7F, 0x01,
+ 0x80, 0x02, 0x7F, 0x00, 0xEF, 0x4D, 0x60, 0x78,
+/*09B0*/0x90, 0x07, 0x26, 0x12, 0x07, 0x35, 0xE0, 0xFF,
+ 0x12, 0x07, 0xFC, 0xEF, 0x12, 0x07, 0x31, 0xE0,
+/*09C0*/0xFF, 0x12, 0x08, 0x08, 0xEF, 0xF0, 0x90, 0x07,
+ 0x22, 0x12, 0x07, 0x35, 0xE0, 0x54, 0x1F, 0xFF,
+/*09D0*/0x12, 0x08, 0x4D, 0xEF, 0xF0, 0x90, 0x07, 0x24,
+ 0x12, 0x07, 0x35, 0xE0, 0x54, 0x1F, 0xFF, 0x12,
+/*09E0*/0x08, 0x59, 0xEF, 0xF0, 0x22, 0x12, 0x07, 0xCC,
+ 0xE4, 0xF0, 0x12, 0x07, 0xD8, 0xE4, 0xF0, 0x12,
+/*09F0*/0x08, 0x81, 0xF5, 0x83, 0xE4, 0xF0, 0x12, 0x08,
+ 0x35, 0x74, 0x14, 0xF0, 0x12, 0x07, 0xE4, 0xE4,
+/*0A00*/0xF0, 0x12, 0x07, 0xF0, 0xE4, 0xF0, 0x12, 0x08,
+ 0x8B, 0xF5, 0x83, 0xE4, 0xF0, 0x12, 0x08, 0x41,
+/*0A10*/0x74, 0x14, 0xF0, 0x12, 0x07, 0xFC, 0xE4, 0xF0,
+ 0x12, 0x08, 0x08, 0xE4, 0xF0, 0x12, 0x08, 0x4D,
+/*0A20*/0xE4, 0xF0, 0x12, 0x08, 0x59, 0x74, 0x14, 0xF0,
+ 0x22, 0x53, 0xF9, 0xF7, 0x75, 0xFC, 0x10, 0xE4,
+/*0A30*/0xF5, 0xFD, 0x75, 0xFE, 0x30, 0xF5, 0xFF, 0xE5,
+ 0xE7, 0x20, 0xE7, 0x03, 0x43, 0xF9, 0x08, 0xE5,
+/*0A40*/0xE6, 0x20, 0xE7, 0x0B, 0x78, 0xFF, 0xE4, 0xF6,
+ 0xD8, 0xFD, 0x53, 0xE6, 0xFE, 0x80, 0x09, 0x78,
+/*0A50*/0x08, 0xE4, 0xF6, 0xD8, 0xFD, 0x53, 0xE6, 0xFE,
+ 0x75, 0x81, 0x80, 0xE4, 0xF5, 0xA8, 0xD2, 0xA8,
+/*0A60*/0xC2, 0xA9, 0xD2, 0xAF, 0xE5, 0xE2, 0x20, 0xE5,
+ 0x05, 0x20, 0xE6, 0x02, 0x80, 0x03, 0x43, 0xE1,
+/*0A70*/0x02, 0xE5, 0xE2, 0x20, 0xE0, 0x0E, 0x90, 0x00,
+ 0x00, 0x7F, 0x00, 0x7E, 0x08, 0xE4, 0xF0, 0xA3,
+/*0A80*/0xDF, 0xFC, 0xDE, 0xFA, 0x02, 0x0A, 0xDB, 0x43,
+ 0xFA, 0x01, 0xC0, 0xE0, 0xC0, 0xF0, 0xC0, 0x83,
+/*0A90*/0xC0, 0x82, 0xC0, 0xD0, 0x12, 0x1C, 0xE7, 0xD0,
+ 0xD0, 0xD0, 0x82, 0xD0, 0x83, 0xD0, 0xF0, 0xD0,
+/*0AA0*/0xE0, 0x53, 0xFA, 0xFE, 0x32, 0x02, 0x1B, 0x55,
+ 0xE4, 0x93, 0xA3, 0xF8, 0xE4, 0x93, 0xA3, 0xF6,
+/*0AB0*/0x08, 0xDF, 0xF9, 0x80, 0x29, 0xE4, 0x93, 0xA3,
+ 0xF8, 0x54, 0x07, 0x24, 0x0C, 0xC8, 0xC3, 0x33,
+/*0AC0*/0xC4, 0x54, 0x0F, 0x44, 0x20, 0xC8, 0x83, 0x40,
+ 0x04, 0xF4, 0x56, 0x80, 0x01, 0x46, 0xF6, 0xDF,
+/*0AD0*/0xE4, 0x80, 0x0B, 0x01, 0x02, 0x04, 0x08, 0x10,
+ 0x20, 0x40, 0x80, 0x90, 0x00, 0x3F, 0xE4, 0x7E,
+/*0AE0*/0x01, 0x93, 0x60, 0xC1, 0xA3, 0xFF, 0x54, 0x3F,
+ 0x30, 0xE5, 0x09, 0x54, 0x1F, 0xFE, 0xE4, 0x93,
+/*0AF0*/0xA3, 0x60, 0x01, 0x0E, 0xCF, 0x54, 0xC0, 0x25,
+ 0xE0, 0x60, 0xAD, 0x40, 0xB8, 0x80, 0xFE, 0x8C,
+/*0B00*/0x64, 0x8D, 0x65, 0x8A, 0x66, 0x8B, 0x67, 0xE4,
+ 0xF5, 0x69, 0xEF, 0x4E, 0x70, 0x03, 0x02, 0x1D,
+/*0B10*/0x55, 0xE4, 0xF5, 0x68, 0xE5, 0x67, 0x45, 0x66,
+ 0x70, 0x32, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x90,
+/*0B20*/0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2, 0xE4,
+ 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0xE4, 0x12,
+/*0B30*/0x08, 0x70, 0x70, 0x29, 0x12, 0x07, 0x2A, 0x75,
+ 0x83, 0x92, 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83,
+/*0B40*/0xC6, 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC8,
+ 0xE4, 0xF0, 0x80, 0x11, 0x90, 0x07, 0x26, 0x12,
+/*0B50*/0x07, 0x35, 0xE4, 0x12, 0x08, 0x70, 0x70, 0x05,
+ 0x12, 0x07, 0x32, 0xE4, 0xF0, 0x12, 0x1D, 0x55,
+/*0B60*/0x12, 0x1E, 0xBF, 0xE5, 0x67, 0x45, 0x66, 0x70,
+ 0x33, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x90, 0xE5,
+/*0B70*/0x41, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2, 0xE5,
+ 0x41, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0x12,
+/*0B80*/0x08, 0x6E, 0x70, 0x29, 0x12, 0x07, 0x2A, 0x75,
+ 0x83, 0x92, 0xE5, 0x40, 0x12, 0x07, 0x29, 0x75,
+/*0B90*/0x83, 0xC6, 0xE5, 0x40, 0x12, 0x07, 0x29, 0x75,
+ 0x83, 0xC8, 0x80, 0x0E, 0x90, 0x07, 0x26, 0x12,
+/*0BA0*/0x07, 0x35, 0x12, 0x08, 0x6E, 0x70, 0x06, 0x12,
+ 0x07, 0x32, 0xE5, 0x40, 0xF0, 0xAF, 0x69, 0x7E,
+/*0BB0*/0x00, 0xAD, 0x67, 0xAC, 0x66, 0x12, 0x04, 0x44,
+ 0x12, 0x07, 0x2A, 0x75, 0x83, 0xCA, 0xE0, 0xD3,
+/*0BC0*/0x94, 0x00, 0x50, 0x0C, 0x05, 0x68, 0xE5, 0x68,
+ 0xC3, 0x94, 0x05, 0x50, 0x03, 0x02, 0x0B, 0x14,
+/*0BD0*/0x22, 0x8C, 0x60, 0x8D, 0x61, 0x12, 0x08, 0xDA,
+ 0x74, 0x20, 0x40, 0x0D, 0x2F, 0xF5, 0x82, 0x74,
+/*0BE0*/0x03, 0x3E, 0xF5, 0x83, 0xE5, 0x3E, 0xF0, 0x80,
+ 0x0B, 0x2F, 0xF5, 0x82, 0x74, 0x03, 0x3E, 0xF5,
+/*0BF0*/0x83, 0xE5, 0x3C, 0xF0, 0xE5, 0x3C, 0xD3, 0x95,
+ 0x3E, 0x40, 0x3C, 0xE5, 0x61, 0x45, 0x60, 0x70,
+/*0C00*/0x10, 0xE9, 0x12, 0x09, 0x04, 0xE5, 0x3E, 0x12,
+ 0x07, 0x68, 0x40, 0x3B, 0x12, 0x08, 0x95, 0x80,
+/*0C10*/0x18, 0xE5, 0x3E, 0xC3, 0x95, 0x38, 0x40, 0x1D,
+ 0x85, 0x3E, 0x38, 0xE5, 0x3E, 0x60, 0x05, 0x85,
+/*0C20*/0x3F, 0x39, 0x80, 0x03, 0x85, 0x39, 0x39, 0x8F,
+ 0x3A, 0x12, 0x08, 0x14, 0xE5, 0x3E, 0x12, 0x07,
+/*0C30*/0xC0, 0xE5, 0x3F, 0xF0, 0x22, 0x80, 0x43, 0xE5,
+ 0x61, 0x45, 0x60, 0x70, 0x19, 0x12, 0x07, 0x5F,
+/*0C40*/0x40, 0x05, 0x12, 0x08, 0x9E, 0x80, 0x27, 0x12,
+ 0x09, 0x0B, 0x12, 0x08, 0x14, 0xE5, 0x42, 0x12,
+/*0C50*/0x07, 0xC0, 0xE5, 0x41, 0xF0, 0x22, 0xE5, 0x3C,
+ 0xC3, 0x95, 0x38, 0x40, 0x1D, 0x85, 0x3C, 0x38,
+/*0C60*/0xE5, 0x3C, 0x60, 0x05, 0x85, 0x3D, 0x39, 0x80,
+ 0x03, 0x85, 0x39, 0x39, 0x8F, 0x3A, 0x12, 0x08,
+/*0C70*/0x14, 0xE5, 0x3C, 0x12, 0x07, 0xC0, 0xE5, 0x3D,
+ 0xF0, 0x22, 0x85, 0x38, 0x38, 0x85, 0x39, 0x39,
+/*0C80*/0x85, 0x3A, 0x3A, 0x12, 0x08, 0x14, 0xE5, 0x38,
+ 0x12, 0x07, 0xC0, 0xE5, 0x39, 0xF0, 0x22, 0x7F,
+/*0C90*/0x06, 0x12, 0x17, 0x31, 0x12, 0x1D, 0x23, 0x12,
+ 0x0E, 0x04, 0x12, 0x0E, 0x33, 0xE0, 0x44, 0x0A,
+/*0CA0*/0xF0, 0x74, 0x8E, 0xFE, 0x12, 0x0E, 0x04, 0x12,
+ 0x0E, 0x0B, 0xEF, 0xF0, 0xE5, 0x28, 0x30, 0xE5,
+/*0CB0*/0x03, 0xD3, 0x80, 0x01, 0xC3, 0x40, 0x05, 0x75,
+ 0x14, 0x20, 0x80, 0x03, 0x75, 0x14, 0x08, 0x12,
+/*0CC0*/0x0E, 0x04, 0x75, 0x83, 0x8A, 0xE5, 0x14, 0xF0,
+ 0xB4, 0xFF, 0x05, 0x75, 0x12, 0x80, 0x80, 0x06,
+/*0CD0*/0xE5, 0x14, 0xC3, 0x13, 0xF5, 0x12, 0xE4, 0xF5,
+ 0x16, 0xF5, 0x7F, 0x12, 0x19, 0x36, 0x12, 0x13,
+/*0CE0*/0xA3, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x50, 0x09,
+ 0x05, 0x16, 0xE5, 0x16, 0xC3, 0x94, 0x14, 0x40,
+/*0CF0*/0xEA, 0xE5, 0xE4, 0x20, 0xE7, 0x28, 0x12, 0x0E,
+ 0x04, 0x75, 0x83, 0xD2, 0xE0, 0x54, 0x08, 0xD3,
+/*0D00*/0x94, 0x00, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02,
+ 0x7F, 0x00, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x40,
+/*0D10*/0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEF,
+ 0x5E, 0x60, 0x03, 0x12, 0x1D, 0xD7, 0xE5, 0x7F,
+/*0D20*/0xC3, 0x94, 0x11, 0x40, 0x14, 0x12, 0x0E, 0x04,
+ 0x75, 0x83, 0xD2, 0xE0, 0x44, 0x80, 0xF0, 0xE5,
+/*0D30*/0xE4, 0x20, 0xE7, 0x0F, 0x12, 0x1D, 0xD7, 0x80,
+ 0x0A, 0x12, 0x0E, 0x04, 0x75, 0x83, 0xD2, 0xE0,
+/*0D40*/0x54, 0x7F, 0xF0, 0x12, 0x1D, 0x23, 0x22, 0x74,
+ 0x8A, 0x85, 0x08, 0x82, 0xF5, 0x83, 0xE5, 0x17,
+/*0D50*/0xF0, 0x12, 0x0E, 0x3A, 0xE4, 0xF0, 0x90, 0x07,
+ 0x02, 0xE0, 0x12, 0x0E, 0x17, 0x75, 0x83, 0x90,
+/*0D60*/0xEF, 0xF0, 0x74, 0x92, 0xFE, 0xE5, 0x08, 0x44,
+ 0x07, 0xFF, 0xF5, 0x82, 0x8E, 0x83, 0xE0, 0x54,
+/*0D70*/0xC0, 0xFD, 0x90, 0x07, 0x03, 0xE0, 0x54, 0x3F,
+ 0x4D, 0x8F, 0x82, 0x8E, 0x83, 0xF0, 0x90, 0x07,
+/*0D80*/0x04, 0xE0, 0x12, 0x0E, 0x17, 0x75, 0x83, 0x82,
+ 0xEF, 0xF0, 0x90, 0x07, 0x05, 0xE0, 0xFF, 0xED,
+/*0D90*/0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0xB4, 0xEF,
+ 0x12, 0x0E, 0x03, 0x75, 0x83, 0x80, 0xE0, 0x54,
+/*0DA0*/0xBF, 0xF0, 0x30, 0x37, 0x0A, 0x12, 0x0E, 0x91,
+ 0x75, 0x83, 0x94, 0xE0, 0x44, 0x80, 0xF0, 0x30,
+/*0DB0*/0x38, 0x0A, 0x12, 0x0E, 0x91, 0x75, 0x83, 0x92,
+ 0xE0, 0x44, 0x80, 0xF0, 0xE5, 0x28, 0x30, 0xE4,
+/*0DC0*/0x1A, 0x20, 0x39, 0x0A, 0x12, 0x0E, 0x04, 0x75,
+ 0x83, 0x88, 0xE0, 0x54, 0x7F, 0xF0, 0x20, 0x3A,
+/*0DD0*/0x0A, 0x12, 0x0E, 0x04, 0x75, 0x83, 0x88, 0xE0,
+ 0x54, 0xBF, 0xF0, 0x74, 0x8C, 0xFE, 0x12, 0x0E,
+/*0DE0*/0x04, 0x8E, 0x83, 0xE0, 0x54, 0x0F, 0x12, 0x0E,
+ 0x03, 0x75, 0x83, 0x86, 0xE0, 0x54, 0xBF, 0xF0,
+/*0DF0*/0xE5, 0x08, 0x44, 0x06, 0x12, 0x0D, 0xFD, 0x75,
+ 0x83, 0x8A, 0xE4, 0xF0, 0x22, 0xF5, 0x82, 0x75,
+/*0E00*/0x83, 0x82, 0xE4, 0xF0, 0xE5, 0x08, 0x44, 0x07,
+ 0xF5, 0x82, 0x22, 0x8E, 0x83, 0xE0, 0xF5, 0x10,
+/*0E10*/0x54, 0xFE, 0xF0, 0xE5, 0x10, 0x44, 0x01, 0xFF,
0xE5, 0x08, 0xFD, 0xED, 0x44, 0x07, 0xF5, 0x82,
-/*0870*/0x22, 0xE5, 0x41, 0xF0, 0xE5, 0x65, 0x64, 0x01,
- 0x45, 0x64, 0x22, 0x7E, 0x00, 0xFB, 0x7A, 0x00,
-/*0880*/0xFD, 0x7C, 0x00, 0x22, 0x74, 0x20, 0x25, 0x11,
- 0xF5, 0x82, 0xE4, 0x34, 0x02, 0x22, 0x74, 0xA0,
-/*0890*/0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34, 0x03, 0x22,
- 0xAF, 0x08, 0x7E, 0x00, 0xEF, 0x44, 0x07, 0xF5,
-/*08A0*/0x82, 0x22, 0x85, 0x3E, 0x42, 0x85, 0x3F, 0x41,
- 0x8F, 0x40, 0x22, 0x85, 0x3C, 0x42, 0x85, 0x3D,
-/*08B0*/0x41, 0x8F, 0x40, 0x22, 0x75, 0x45, 0x3F, 0x90,
- 0x07, 0x20, 0xE4, 0xF0, 0xA3, 0x22, 0xF5, 0x83,
-/*08C0*/0xE5, 0x32, 0xF0, 0x05, 0x6E, 0xE5, 0x6E, 0xC3,
- 0x94, 0x40, 0x22, 0xF0, 0xE5, 0x08, 0x44, 0x06,
-/*08D0*/0xF5, 0x82, 0x22, 0x74, 0x00, 0x25, 0x6E, 0xF5,
- 0x82, 0xE4, 0x34, 0x00, 0xF5, 0x83, 0x22, 0xE5,
-/*08E0*/0x6D, 0x45, 0x6C, 0x90, 0x07, 0x2F, 0x22, 0xE4,
- 0xF9, 0xE5, 0x3C, 0xD3, 0x95, 0x3E, 0x22, 0x74,
-/*08F0*/0x80, 0x2E, 0xF5, 0x82, 0xE4, 0x34, 0x02, 0xF5,
- 0x83, 0xE0, 0x22, 0x74, 0xA0, 0x2E, 0xF5, 0x82,
-/*0900*/0xE4, 0x34, 0x02, 0xF5, 0x83, 0xE0, 0x22, 0x74,
- 0x80, 0x25, 0x6E, 0xF5, 0x82, 0xE4, 0x34, 0x00,
-/*0910*/0x22, 0x25, 0x42, 0xFD, 0xE4, 0x33, 0xFC, 0x22,
- 0x85, 0x42, 0x42, 0x85, 0x41, 0x41, 0x85, 0x40,
-/*0920*/0x40, 0x22, 0xED, 0x4C, 0x60, 0x03, 0x02, 0x09,
- 0xF2, 0xEF, 0x4E, 0x70, 0x37, 0x90, 0x07, 0x26,
-/*0930*/0x12, 0x07, 0x96, 0xE0, 0xFD, 0x12, 0x07, 0xD9,
- 0xED, 0xF0, 0x90, 0x07, 0x28, 0x12, 0x07, 0x96,
-/*0940*/0xE0, 0xFD, 0x12, 0x07, 0xE5, 0xED, 0xF0, 0x12,
- 0x07, 0x93, 0xE0, 0x54, 0x1F, 0xFD, 0x12, 0x08,
-/*0950*/0x84, 0xF5, 0x83, 0xED, 0xF0, 0x90, 0x07, 0x24,
- 0x12, 0x07, 0x96, 0xE0, 0x54, 0x1F, 0xFD, 0x12,
-/*0960*/0x08, 0x38, 0xED, 0xF0, 0xEF, 0x64, 0x04, 0x4E,
- 0x70, 0x37, 0x90, 0x07, 0x26, 0x12, 0x07, 0x96,
-/*0970*/0xE0, 0xFD, 0x12, 0x07, 0xF1, 0xED, 0xF0, 0x90,
- 0x07, 0x28, 0x12, 0x07, 0x96, 0xE0, 0xFD, 0x12,
-/*0980*/0x07, 0xFD, 0xED, 0xF0, 0x12, 0x07, 0x93, 0xE0,
- 0x54, 0x1F, 0xFD, 0x12, 0x08, 0x8E, 0xF5, 0x83,
-/*0990*/0xED, 0xF0, 0x90, 0x07, 0x24, 0x12, 0x07, 0x96,
- 0xE0, 0x54, 0x1F, 0xFD, 0x12, 0x08, 0x44, 0xED,
-/*09A0*/0xF0, 0xEF, 0x64, 0x01, 0x4E, 0x70, 0x04, 0x7D,
- 0x01, 0x80, 0x02, 0x7D, 0x00, 0xEF, 0x64, 0x02,
-/*09B0*/0x4E, 0x70, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F,
- 0x00, 0xEF, 0x4D, 0x60, 0x78, 0x90, 0x07, 0x26,
-/*09C0*/0x12, 0x07, 0x42, 0xE0, 0xFF, 0x12, 0x08, 0x09,
- 0xEF, 0x12, 0x07, 0x3E, 0xE0, 0xFF, 0x12, 0x08,
-/*09D0*/0x15, 0xEF, 0xF0, 0x90, 0x07, 0x22, 0x12, 0x07,
- 0x42, 0xE0, 0x54, 0x1F, 0xFF, 0x12, 0x08, 0x50,
-/*09E0*/0xEF, 0xF0, 0x90, 0x07, 0x24, 0x12, 0x07, 0x42,
- 0xE0, 0x54, 0x1F, 0xFF, 0x12, 0x08, 0x5C, 0xEF,
-/*09F0*/0xF0, 0x22, 0x12, 0x07, 0xD9, 0xE4, 0xF0, 0x12,
- 0x07, 0xE5, 0xE4, 0xF0, 0x12, 0x08, 0x84, 0xF5,
-/*0A00*/0x83, 0xE4, 0xF0, 0x12, 0x08, 0x38, 0x74, 0x14,
- 0xF0, 0x12, 0x07, 0xF1, 0xE4, 0xF0, 0x12, 0x07,
-/*0A10*/0xFD, 0xE4, 0xF0, 0x12, 0x08, 0x8E, 0xF5, 0x83,
- 0xE4, 0xF0, 0x12, 0x08, 0x44, 0x74, 0x14, 0xF0,
-/*0A20*/0x12, 0x08, 0x09, 0xE4, 0xF0, 0x12, 0x08, 0x15,
- 0xE4, 0xF0, 0x12, 0x08, 0x50, 0xE4, 0xF0, 0x12,
-/*0A30*/0x08, 0x5C, 0x74, 0x14, 0xF0, 0x22, 0x8C, 0x64,
- 0x8D, 0x65, 0x8A, 0x66, 0x8B, 0x67, 0xE4, 0xF5,
-/*0A40*/0x69, 0xEF, 0x4E, 0x70, 0x03, 0x02, 0x1D, 0x26,
- 0xE4, 0xF5, 0x68, 0xE5, 0x67, 0x45, 0x66, 0x70,
-/*0A50*/0x32, 0x12, 0x07, 0x37, 0x75, 0x83, 0x90, 0xE4,
- 0x12, 0x07, 0x36, 0x75, 0x83, 0xC2, 0xE4, 0x12,
-/*0A60*/0x07, 0x36, 0x75, 0x83, 0xC4, 0xE4, 0x12, 0x08,
- 0x73, 0x70, 0x29, 0x12, 0x07, 0x37, 0x75, 0x83,
-/*0A70*/0x92, 0xE4, 0x12, 0x07, 0x36, 0x75, 0x83, 0xC6,
- 0xE4, 0x12, 0x07, 0x36, 0x75, 0x83, 0xC8, 0xE4,
-/*0A80*/0xF0, 0x80, 0x11, 0x90, 0x07, 0x26, 0x12, 0x07,
- 0x42, 0xE4, 0x12, 0x08, 0x73, 0x70, 0x05, 0x12,
-/*0A90*/0x07, 0x3F, 0xE4, 0xF0, 0x12, 0x1D, 0x26, 0x12,
- 0x1E, 0x90, 0xE5, 0x67, 0x45, 0x66, 0x70, 0x33,
-/*0AA0*/0x12, 0x07, 0x37, 0x75, 0x83, 0x90, 0xE5, 0x41,
- 0x12, 0x07, 0x36, 0x75, 0x83, 0xC2, 0xE5, 0x41,
-/*0AB0*/0x12, 0x07, 0x36, 0x75, 0x83, 0xC4, 0x12, 0x08,
- 0x71, 0x70, 0x29, 0x12, 0x07, 0x37, 0x75, 0x83,
-/*0AC0*/0x92, 0xE5, 0x40, 0x12, 0x07, 0x36, 0x75, 0x83,
- 0xC6, 0xE5, 0x40, 0x12, 0x07, 0x36, 0x75, 0x83,
-/*0AD0*/0xC8, 0x80, 0x0E, 0x90, 0x07, 0x26, 0x12, 0x07,
- 0x42, 0x12, 0x08, 0x71, 0x70, 0x06, 0x12, 0x07,
-/*0AE0*/0x3F, 0xE5, 0x40, 0xF0, 0xAF, 0x69, 0x7E, 0x00,
- 0xAD, 0x67, 0xAC, 0x66, 0x12, 0x04, 0x49, 0x12,
-/*0AF0*/0x07, 0x37, 0x75, 0x83, 0xCA, 0xE0, 0xD3, 0x94,
- 0x00, 0x50, 0x0C, 0x05, 0x68, 0xE5, 0x68, 0xC3,
-/*0B00*/0x94, 0x05, 0x50, 0x03, 0x02, 0x0A, 0x4B, 0x22,
- 0x53, 0xF9, 0xF7, 0x75, 0xFC, 0x10, 0xE4, 0xF5,
-/*0B10*/0xFD, 0x75, 0xFE, 0x30, 0xF5, 0xFF, 0x43, 0xF9,
- 0x08, 0xE5, 0xE6, 0x20, 0xE7, 0x0B, 0x78, 0xFF,
-/*0B20*/0xE4, 0xF6, 0xD8, 0xFD, 0x53, 0xE6, 0xFE, 0x80,
- 0x09, 0x78, 0x08, 0xE4, 0xF6, 0xD8, 0xFD, 0x53,
-/*0B30*/0xE6, 0xFE, 0x75, 0x81, 0x80, 0xE4, 0xF5, 0xA8,
- 0xD2, 0xA8, 0xC2, 0xA9, 0xD2, 0xAF, 0xE5, 0xE2,
-/*0B40*/0x20, 0xE5, 0x05, 0x20, 0xE6, 0x02, 0x80, 0x03,
- 0x43, 0xE1, 0x02, 0xE5, 0xE2, 0x20, 0xE0, 0x0E,
-/*0B50*/0x90, 0x00, 0x00, 0x7F, 0x00, 0x7E, 0x08, 0xE4,
- 0xF0, 0xA3, 0xDF, 0xFC, 0xDE, 0xFA, 0x02, 0x0B,
-/*0B60*/0xB5, 0x43, 0xFA, 0x01, 0xC0, 0xE0, 0xC0, 0xF0,
- 0xC0, 0x83, 0xC0, 0x82, 0xC0, 0xD0, 0x12, 0x11,
-/*0B70*/0x18, 0xD0, 0xD0, 0xD0, 0x82, 0xD0, 0x83, 0xD0,
- 0xF0, 0xD0, 0xE0, 0x53, 0xFA, 0xFE, 0x32, 0x02,
-/*0B80*/0x1B, 0x2E, 0xE4, 0x93, 0xA3, 0xF8, 0xE4, 0x93,
- 0xA3, 0xF6, 0x08, 0xDF, 0xF9, 0x80, 0x29, 0xE4,
-/*0B90*/0x93, 0xA3, 0xF8, 0x54, 0x07, 0x24, 0x0C, 0xC8,
- 0xC3, 0x33, 0xC4, 0x54, 0x0F, 0x44, 0x20, 0xC8,
-/*0BA0*/0x83, 0x40, 0x04, 0xF4, 0x56, 0x80, 0x01, 0x46,
- 0xF6, 0xDF, 0xE4, 0x80, 0x0B, 0x01, 0x02, 0x04,
-/*0BB0*/0x08, 0x10, 0x20, 0x40, 0x80, 0x90, 0x0F, 0xFF,
- 0xE4, 0x7E, 0x01, 0x93, 0x60, 0xC1, 0xA3, 0xFF,
-/*0BC0*/0x54, 0x3F, 0x30, 0xE5, 0x09, 0x54, 0x1F, 0xFE,
- 0xE4, 0x93, 0xA3, 0x60, 0x01, 0x0E, 0xCF, 0x54,
-/*0BD0*/0xC0, 0x25, 0xE0, 0x60, 0xAD, 0x40, 0xB8, 0x80,
- 0xFE, 0x8C, 0x60, 0x8D, 0x61, 0x12, 0x08, 0xE7,
-/*0BE0*/0x74, 0x20, 0x40, 0x0D, 0x2F, 0xF5, 0x82, 0x74,
- 0x03, 0x3E, 0xF5, 0x83, 0xE5, 0x3E, 0xF0, 0x80,
-/*0BF0*/0x0B, 0x2F, 0xF5, 0x82, 0x74, 0x03, 0x3E, 0xF5,
- 0x83, 0xE5, 0x3C, 0xF0, 0xE5, 0x3C, 0xD3, 0x95,
-/*0C00*/0x3E, 0x40, 0x3C, 0xE5, 0x61, 0x45, 0x60, 0x70,
- 0x10, 0xE9, 0x12, 0x09, 0x11, 0xE5, 0x3E, 0x12,
-/*0C10*/0x07, 0x75, 0x40, 0x3B, 0x12, 0x08, 0xA2, 0x80,
- 0x18, 0xE5, 0x3E, 0xC3, 0x95, 0x38, 0x40, 0x1D,
-/*0C20*/0x85, 0x3E, 0x38, 0xE5, 0x3E, 0x60, 0x05, 0x85,
- 0x3F, 0x39, 0x80, 0x03, 0x85, 0x39, 0x39, 0x8F,
-/*0C30*/0x3A, 0x12, 0x08, 0x21, 0xE5, 0x3E, 0x12, 0x07,
- 0xCD, 0xE5, 0x3F, 0xF0, 0x22, 0x80, 0x43, 0xE5,
-/*0C40*/0x61, 0x45, 0x60, 0x70, 0x19, 0x12, 0x07, 0x6C,
- 0x40, 0x05, 0x12, 0x08, 0xAB, 0x80, 0x27, 0x12,
-/*0C50*/0x09, 0x18, 0x12, 0x08, 0x21, 0xE5, 0x42, 0x12,
- 0x07, 0xCD, 0xE5, 0x41, 0xF0, 0x22, 0xE5, 0x3C,
-/*0C60*/0xC3, 0x95, 0x38, 0x40, 0x1D, 0x85, 0x3C, 0x38,
- 0xE5, 0x3C, 0x60, 0x05, 0x85, 0x3D, 0x39, 0x80,
-/*0C70*/0x03, 0x85, 0x39, 0x39, 0x8F, 0x3A, 0x12, 0x08,
- 0x21, 0xE5, 0x3C, 0x12, 0x07, 0xCD, 0xE5, 0x3D,
-/*0C80*/0xF0, 0x22, 0x85, 0x38, 0x38, 0x85, 0x39, 0x39,
- 0x85, 0x3A, 0x3A, 0x12, 0x08, 0x21, 0xE5, 0x38,
-/*0C90*/0x12, 0x07, 0xCD, 0xE5, 0x39, 0xF0, 0x22, 0x7F,
- 0x06, 0x12, 0x16, 0xB3, 0x12, 0x1C, 0xF4, 0x12,
-/*0CA0*/0x0E, 0xB7, 0x12, 0x0E, 0xE6, 0xE0, 0x44, 0x0A,
- 0xF0, 0x74, 0x8E, 0xFE, 0x12, 0x0E, 0xB7, 0x12,
-/*0CB0*/0x0E, 0xBE, 0xEF, 0xF0, 0xE5, 0x28, 0x30, 0xE5,
- 0x03, 0xD3, 0x80, 0x01, 0xC3, 0x40, 0x05, 0x75,
-/*0CC0*/0x14, 0x20, 0x80, 0x03, 0x75, 0x14, 0x08, 0x12,
- 0x0E, 0xB7, 0x75, 0x83, 0x8A, 0xE5, 0x14, 0xF0,
-/*0CD0*/0xB4, 0xFF, 0x05, 0x75, 0x12, 0x80, 0x80, 0x06,
- 0xE5, 0x14, 0xC3, 0x13, 0xF5, 0x12, 0xE4, 0xF5,
-/*0CE0*/0x16, 0xF5, 0x7F, 0x12, 0x18, 0xB8, 0x12, 0x13,
- 0xA3, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x50, 0x09,
-/*0CF0*/0x05, 0x16, 0xE5, 0x16, 0xC3, 0x94, 0x14, 0x40,
- 0xEA, 0xE5, 0xE4, 0x20, 0xE7, 0x28, 0x12, 0x0E,
-/*0D00*/0xB7, 0x75, 0x83, 0xD2, 0xE0, 0x54, 0x08, 0xD3,
- 0x94, 0x00, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02,
-/*0D10*/0x7F, 0x00, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x40,
- 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEF,
-/*0D20*/0x5E, 0x60, 0x03, 0x12, 0x1D, 0xA8, 0xE5, 0x7F,
- 0xC3, 0x94, 0x11, 0x40, 0x14, 0x12, 0x0E, 0xB7,
-/*0D30*/0x75, 0x83, 0xD2, 0xE0, 0x44, 0x80, 0xF0, 0xE5,
- 0xE4, 0x20, 0xE7, 0x0F, 0x12, 0x1D, 0xA8, 0x80,
-/*0D40*/0x0A, 0x12, 0x0E, 0xB7, 0x75, 0x83, 0xD2, 0xE0,
- 0x54, 0x7F, 0xF0, 0x12, 0x1C, 0xF4, 0x22, 0x74,
-/*0D50*/0x8A, 0x85, 0x08, 0x82, 0xF5, 0x83, 0xE5, 0x17,
- 0xF0, 0x12, 0x0E, 0xED, 0xE4, 0xF0, 0x90, 0x07,
-/*0D60*/0x02, 0xE0, 0x12, 0x0E, 0xCA, 0x75, 0x83, 0x90,
- 0xEF, 0xF0, 0x74, 0x92, 0xFE, 0xE5, 0x08, 0x44,
-/*0D70*/0x07, 0xFF, 0xF5, 0x82, 0x8E, 0x83, 0xE0, 0x54,
- 0xC0, 0xFD, 0x90, 0x07, 0x03, 0xE0, 0x54, 0x3F,
-/*0D80*/0x4D, 0x8F, 0x82, 0x8E, 0x83, 0xF0, 0x90, 0x07,
- 0x04, 0xE0, 0x12, 0x0E, 0xCA, 0x75, 0x83, 0x82,
-/*0D90*/0xEF, 0xF0, 0x90, 0x07, 0x05, 0xE0, 0xFF, 0xED,
- 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0xB4, 0xEF,
-/*0DA0*/0x12, 0x0E, 0xB6, 0x75, 0x83, 0x80, 0xE0, 0x54,
- 0xBF, 0xF0, 0x30, 0x37, 0x0A, 0x12, 0x0F, 0x44,
-/*0DB0*/0x75, 0x83, 0x94, 0xE0, 0x44, 0x80, 0xF0, 0x30,
- 0x38, 0x0A, 0x12, 0x0F, 0x44, 0x75, 0x83, 0x92,
-/*0DC0*/0xE0, 0x44, 0x80, 0xF0, 0xE5, 0x28, 0x30, 0xE4,
- 0x1A, 0x20, 0x39, 0x0A, 0x12, 0x0E, 0xB7, 0x75,
-/*0DD0*/0x83, 0x88, 0xE0, 0x54, 0x7F, 0xF0, 0x20, 0x3A,
- 0x0A, 0x12, 0x0E, 0xB7, 0x75, 0x83, 0x88, 0xE0,
-/*0DE0*/0x54, 0xBF, 0xF0, 0x74, 0x8C, 0xFE, 0x12, 0x0E,
- 0xB7, 0x8E, 0x83, 0xE0, 0x54, 0x0F, 0x12, 0x0E,
-/*0DF0*/0xB6, 0x75, 0x83, 0x86, 0xE0, 0x54, 0xBF, 0xF0,
- 0xE5, 0x08, 0x44, 0x06, 0x12, 0x0E, 0xB0, 0x75,
-/*0E00*/0x83, 0x8A, 0xE4, 0xF0, 0x22, 0xEF, 0x4E, 0x70,
- 0x26, 0x12, 0x07, 0xD9, 0xE0, 0xFD, 0x90, 0x07,
-/*0E10*/0x26, 0x12, 0x07, 0x88, 0x12, 0x07, 0xE5, 0xE0,
- 0xFD, 0x90, 0x07, 0x28, 0x12, 0x07, 0x88, 0x12,
-/*0E20*/0x08, 0x84, 0x12, 0x07, 0x7F, 0x12, 0x08, 0x38,
- 0xE0, 0x90, 0x07, 0x24, 0x12, 0x07, 0x85, 0xEF,
-/*0E30*/0x64, 0x04, 0x4E, 0x70, 0x29, 0x12, 0x07, 0xF1,
- 0xE0, 0xFD, 0x90, 0x07, 0x26, 0x12, 0x07, 0x88,
-/*0E40*/0x12, 0x07, 0xFD, 0xE0, 0xFD, 0x90, 0x07, 0x28,
- 0x12, 0x07, 0x88, 0x12, 0x08, 0x8E, 0x12, 0x07,
-/*0E50*/0x7F, 0x12, 0x08, 0x44, 0xE0, 0x54, 0x1F, 0xFD,
- 0x90, 0x07, 0x24, 0x12, 0x07, 0x88, 0xEF, 0x64,
-/*0E60*/0x01, 0x4E, 0x70, 0x04, 0x7D, 0x01, 0x80, 0x02,
- 0x7D, 0x00, 0xEF, 0x64, 0x02, 0x4E, 0x70, 0x04,
-/*0E70*/0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xEF, 0x4D,
- 0x60, 0x35, 0x12, 0x08, 0x09, 0xE0, 0xFF, 0x90,
-/*0E80*/0x07, 0x26, 0x12, 0x07, 0x96, 0xEF, 0xF0, 0x12,
- 0x08, 0x15, 0xE0, 0xFF, 0x90, 0x07, 0x28, 0x12,
-/*0E90*/0x07, 0x96, 0xEF, 0xF0, 0x12, 0x08, 0x50, 0xE0,
- 0x54, 0x1F, 0xFF, 0x12, 0x07, 0x93, 0xEF, 0xF0,
-/*0EA0*/0x12, 0x08, 0x5C, 0xE0, 0x54, 0x1F, 0xFF, 0x90,
- 0x07, 0x24, 0x12, 0x07, 0x96, 0xEF, 0xF0, 0x22,
-/*0EB0*/0xF5, 0x82, 0x75, 0x83, 0x82, 0xE4, 0xF0, 0xE5,
- 0x08, 0x44, 0x07, 0xF5, 0x82, 0x22, 0x8E, 0x83,
-/*0EC0*/0xE0, 0xF5, 0x10, 0x54, 0xFE, 0xF0, 0xE5, 0x10,
- 0x44, 0x01, 0xFF, 0xE5, 0x08, 0xFD, 0xED, 0x44,
-/*0ED0*/0x07, 0xF5, 0x82, 0x22, 0xE5, 0x15, 0xC4, 0x54,
- 0x07, 0xFF, 0xE5, 0x08, 0xFD, 0xED, 0x44, 0x08,
-/*0EE0*/0xF5, 0x82, 0x75, 0x83, 0x82, 0x22, 0x75, 0x83,
- 0x80, 0xE0, 0x44, 0x40, 0xF0, 0xE5, 0x08, 0x44,
-/*0EF0*/0x08, 0xF5, 0x82, 0x75, 0x83, 0x8A, 0x22, 0xE5,
- 0x16, 0x25, 0xE0, 0x25, 0xE0, 0x24, 0x88, 0xF5,
-/*0F00*/0x82, 0xE4, 0x34, 0x1A, 0xF5, 0x83, 0xE4, 0x93,
- 0xF5, 0x0D, 0x22, 0x43, 0xE1, 0x10, 0x43, 0xE1,
-/*0F10*/0x80, 0x53, 0xE1, 0xFD, 0x85, 0xE1, 0x10, 0x22,
- 0xE5, 0x16, 0x25, 0xE0, 0x25, 0xE0, 0x24, 0x8B,
-/*0F20*/0xF5, 0x82, 0xE4, 0x34, 0x1A, 0xF5, 0x83, 0xE4,
- 0x93, 0x22, 0x85, 0x55, 0x82, 0x85, 0x54, 0x83,
-/*0F30*/0xE5, 0x15, 0xF0, 0x22, 0xE5, 0xE2, 0x54, 0x20,
- 0xD3, 0x94, 0x00, 0x22, 0xE5, 0xE2, 0x54, 0x40,
-/*0F40*/0xD3, 0x94, 0x00, 0x22, 0xE5, 0x08, 0x44, 0x06,
- 0xF5, 0x82, 0x22, 0xFD, 0xE5, 0x08, 0xFB, 0xEB,
-/*0F50*/0x44, 0x07, 0xF5, 0x82, 0x22, 0xE4, 0xF5, 0x53,
- 0x12, 0x0F, 0x34, 0x40, 0x04, 0x7F, 0x01, 0x80,
-/*0F60*/0x02, 0x7F, 0x00, 0x12, 0x0F, 0x3C, 0x40, 0x04,
- 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F,
-/*0F70*/0x70, 0x03, 0x02, 0x0F, 0xF7, 0x85, 0xE1, 0x10,
- 0x43, 0xE1, 0x02, 0x53, 0xE1, 0x0F, 0x85, 0xE1,
-/*0F80*/0x10, 0xE4, 0xF5, 0x51, 0xE5, 0xE3, 0x54, 0x3F,
- 0xF5, 0x52, 0x12, 0x0F, 0x3C, 0x40, 0x1D, 0xAD,
-/*0F90*/0x52, 0xAF, 0x51, 0x12, 0x1C, 0xC0, 0xEF, 0x60,
- 0x08, 0x85, 0xE1, 0x10, 0x43, 0xE1, 0x40, 0x80,
-/*0FA0*/0x0B, 0x53, 0xE1, 0xBF, 0x12, 0x0F, 0x0B, 0x12,
- 0x00, 0x06, 0x80, 0xFB, 0xE5, 0xE3, 0x54, 0x3F,
-/*0FB0*/0xF5, 0x51, 0xE5, 0xE4, 0x54, 0x3F, 0xF5, 0x52,
- 0x12, 0x0F, 0x34, 0x40, 0x1D, 0xAD, 0x52, 0xAF,
-/*0FC0*/0x51, 0x12, 0x1C, 0xC0, 0xEF, 0x60, 0x08, 0x85,
- 0xE1, 0x10, 0x43, 0xE1, 0x20, 0x80, 0x0B, 0x53,
-/*0FD0*/0xE1, 0xDF, 0x12, 0x0F, 0x0B, 0x12, 0x00, 0x06,
- 0x80, 0xFB, 0x12, 0x0F, 0x34, 0x40, 0x04, 0x7F,
-/*0FE0*/0x01, 0x80, 0x02, 0x7F, 0x00, 0x12, 0x0F, 0x3C,
- 0x40, 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00,
-/*0FF0*/0xEE, 0x4F, 0x60, 0x03, 0x12, 0x0F, 0x0E, 0x22,
- 0x01, 0x1F, 0x00, 0x01, 0x04, 0x20, 0x00, 0x00,
+/*0E20*/0x22, 0xE5, 0x15, 0xC4, 0x54, 0x07, 0xFF, 0xE5,
+ 0x08, 0xFD, 0xED, 0x44, 0x08, 0xF5, 0x82, 0x75,
+/*0E30*/0x83, 0x82, 0x22, 0x75, 0x83, 0x80, 0xE0, 0x44,
+ 0x40, 0xF0, 0xE5, 0x08, 0x44, 0x08, 0xF5, 0x82,
+/*0E40*/0x75, 0x83, 0x8A, 0x22, 0xE5, 0x16, 0x25, 0xE0,
+ 0x25, 0xE0, 0x24, 0xAF, 0xF5, 0x82, 0xE4, 0x34,
+/*0E50*/0x1A, 0xF5, 0x83, 0xE4, 0x93, 0xF5, 0x0D, 0x22,
+ 0x43, 0xE1, 0x10, 0x43, 0xE1, 0x80, 0x53, 0xE1,
+/*0E60*/0xFD, 0x85, 0xE1, 0x10, 0x22, 0xE5, 0x16, 0x25,
+ 0xE0, 0x25, 0xE0, 0x24, 0xB2, 0xF5, 0x82, 0xE4,
+/*0E70*/0x34, 0x1A, 0xF5, 0x83, 0xE4, 0x93, 0x22, 0x85,
+ 0x55, 0x82, 0x85, 0x54, 0x83, 0xE5, 0x15, 0xF0,
+/*0E80*/0x22, 0xE5, 0xE2, 0x54, 0x20, 0xD3, 0x94, 0x00,
+ 0x22, 0xE5, 0xE2, 0x54, 0x40, 0xD3, 0x94, 0x00,
+/*0E90*/0x22, 0xE5, 0x08, 0x44, 0x06, 0xF5, 0x82, 0x22,
+ 0xFD, 0xE5, 0x08, 0xFB, 0xEB, 0x44, 0x07, 0xF5,
+/*0EA0*/0x82, 0x22, 0x53, 0xF9, 0xF7, 0x75, 0xFE, 0x30,
+ 0x22, 0xEF, 0x4E, 0x70, 0x26, 0x12, 0x07, 0xCC,
+/*0EB0*/0xE0, 0xFD, 0x90, 0x07, 0x26, 0x12, 0x07, 0x7B,
+ 0x12, 0x07, 0xD8, 0xE0, 0xFD, 0x90, 0x07, 0x28,
+/*0EC0*/0x12, 0x07, 0x7B, 0x12, 0x08, 0x81, 0x12, 0x07,
+ 0x72, 0x12, 0x08, 0x35, 0xE0, 0x90, 0x07, 0x24,
+/*0ED0*/0x12, 0x07, 0x78, 0xEF, 0x64, 0x04, 0x4E, 0x70,
+ 0x29, 0x12, 0x07, 0xE4, 0xE0, 0xFD, 0x90, 0x07,
+/*0EE0*/0x26, 0x12, 0x07, 0x7B, 0x12, 0x07, 0xF0, 0xE0,
+ 0xFD, 0x90, 0x07, 0x28, 0x12, 0x07, 0x7B, 0x12,
+/*0EF0*/0x08, 0x8B, 0x12, 0x07, 0x72, 0x12, 0x08, 0x41,
+ 0xE0, 0x54, 0x1F, 0xFD, 0x90, 0x07, 0x24, 0x12,
+/*0F00*/0x07, 0x7B, 0xEF, 0x64, 0x01, 0x4E, 0x70, 0x04,
+ 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00, 0xEF, 0x64,
+/*0F10*/0x02, 0x4E, 0x70, 0x04, 0x7F, 0x01, 0x80, 0x02,
+ 0x7F, 0x00, 0xEF, 0x4D, 0x60, 0x35, 0x12, 0x07,
+/*0F20*/0xFC, 0xE0, 0xFF, 0x90, 0x07, 0x26, 0x12, 0x07,
+ 0x89, 0xEF, 0xF0, 0x12, 0x08, 0x08, 0xE0, 0xFF,
+/*0F30*/0x90, 0x07, 0x28, 0x12, 0x07, 0x89, 0xEF, 0xF0,
+ 0x12, 0x08, 0x4D, 0xE0, 0x54, 0x1F, 0xFF, 0x12,
+/*0F40*/0x07, 0x86, 0xEF, 0xF0, 0x12, 0x08, 0x59, 0xE0,
+ 0x54, 0x1F, 0xFF, 0x90, 0x07, 0x24, 0x12, 0x07,
+/*0F50*/0x89, 0xEF, 0xF0, 0x22, 0xE4, 0xF5, 0x53, 0x12,
+ 0x0E, 0x81, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02,
+/*0F60*/0x7F, 0x00, 0x12, 0x0E, 0x89, 0x40, 0x04, 0x7E,
+ 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F, 0x70,
+/*0F70*/0x03, 0x02, 0x0F, 0xF6, 0x85, 0xE1, 0x10, 0x43,
+ 0xE1, 0x02, 0x53, 0xE1, 0x0F, 0x85, 0xE1, 0x10,
+/*0F80*/0xE4, 0xF5, 0x51, 0xE5, 0xE3, 0x54, 0x3F, 0xF5,
+ 0x52, 0x12, 0x0E, 0x89, 0x40, 0x1D, 0xAD, 0x52,
+/*0F90*/0xAF, 0x51, 0x12, 0x11, 0x18, 0xEF, 0x60, 0x08,
+ 0x85, 0xE1, 0x10, 0x43, 0xE1, 0x40, 0x80, 0x0B,
+/*0FA0*/0x53, 0xE1, 0xBF, 0x12, 0x0E, 0x58, 0x12, 0x00,
+ 0x06, 0x80, 0xFB, 0xE5, 0xE3, 0x54, 0x3F, 0xF5,
+/*0FB0*/0x51, 0xE5, 0xE4, 0x54, 0x3F, 0xF5, 0x52, 0x12,
+ 0x0E, 0x81, 0x40, 0x1D, 0xAD, 0x52, 0xAF, 0x51,
+/*0FC0*/0x12, 0x11, 0x18, 0xEF, 0x60, 0x08, 0x85, 0xE1,
+ 0x10, 0x43, 0xE1, 0x20, 0x80, 0x0B, 0x53, 0xE1,
+/*0FD0*/0xDF, 0x12, 0x0E, 0x58, 0x12, 0x00, 0x06, 0x80,
+ 0xFB, 0x12, 0x0E, 0x81, 0x40, 0x04, 0x7F, 0x01,
+/*0FE0*/0x80, 0x02, 0x7F, 0x00, 0x12, 0x0E, 0x89, 0x40,
+ 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE,
+/*0FF0*/0x4F, 0x60, 0x03, 0x12, 0x0E, 0x5B, 0x22, 0x12,
+ 0x0E, 0x21, 0xEF, 0xF0, 0x12, 0x10, 0x91, 0x22,
/*1000*/0x02, 0x11, 0x00, 0x02, 0x10, 0x40, 0x02, 0x10,
0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-/*1010*/0x01, 0x1F, 0x01, 0x1F, 0xE4, 0xF5, 0x57, 0x12,
- 0x16, 0x3F, 0x12, 0x15, 0xC6, 0xE4, 0x12, 0x10,
+/*1010*/0x01, 0x20, 0x01, 0x20, 0xE4, 0xF5, 0x57, 0x12,
+ 0x16, 0xBD, 0x12, 0x16, 0x44, 0xE4, 0x12, 0x10,
/*1020*/0x56, 0x12, 0x14, 0xB7, 0x90, 0x07, 0x26, 0x12,
- 0x07, 0x42, 0xE4, 0x12, 0x07, 0x3E, 0xE4, 0xF0,
+ 0x07, 0x35, 0xE4, 0x12, 0x07, 0x31, 0xE4, 0xF0,
/*1030*/0x12, 0x10, 0x56, 0x12, 0x14, 0xB7, 0x90, 0x07,
- 0x26, 0x12, 0x07, 0x42, 0xE5, 0x41, 0x12, 0x07,
-/*1040*/0x3E, 0xE5, 0x40, 0xF0, 0xAF, 0x57, 0x7E, 0x00,
- 0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x49, 0xAF,
+ 0x26, 0x12, 0x07, 0x35, 0xE5, 0x41, 0x12, 0x07,
+/*1040*/0x31, 0xE5, 0x40, 0xF0, 0xAF, 0x57, 0x7E, 0x00,
+ 0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x44, 0xAF,
/*1050*/0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE, 0xFF, 0x90,
0x07, 0x20, 0xA3, 0xE0, 0xFD, 0xE4, 0xF5, 0x56,
/*1060*/0xF5, 0x40, 0xFE, 0xFC, 0xAB, 0x56, 0xFA, 0x12,
0x11, 0x51, 0x7F, 0x0F, 0x7D, 0x18, 0xE4, 0xF5,
/*1070*/0x56, 0xF5, 0x40, 0xFE, 0xFC, 0xAB, 0x56, 0xFA,
0x12, 0x15, 0x41, 0xAF, 0x56, 0x7E, 0x00, 0x12,
-/*1080*/0x1A, 0xD8, 0xE4, 0xFF, 0xF5, 0x56, 0x7D, 0x1F,
+/*1080*/0x1A, 0xFF, 0xE4, 0xFF, 0xF5, 0x56, 0x7D, 0x1F,
0xF5, 0x40, 0xFE, 0xFC, 0xAB, 0x56, 0xFA, 0x22,
/*1090*/0x22, 0xE4, 0xF5, 0x55, 0xE5, 0x08, 0xFD, 0x74,
0xA0, 0xF5, 0x56, 0xED, 0x44, 0x07, 0xF5, 0x57,
/*10A0*/0xE5, 0x28, 0x30, 0xE5, 0x03, 0xD3, 0x80, 0x01,
0xC3, 0x40, 0x05, 0x7F, 0x28, 0xEF, 0x80, 0x04,
/*10B0*/0x7F, 0x14, 0xEF, 0xC3, 0x13, 0xF5, 0x54, 0xE4,
- 0xF9, 0x12, 0x0E, 0xCB, 0x75, 0x83, 0x8E, 0xE0,
+ 0xF9, 0x12, 0x0E, 0x18, 0x75, 0x83, 0x8E, 0xE0,
/*10C0*/0xF5, 0x10, 0xCE, 0xEF, 0xCE, 0xEE, 0xD3, 0x94,
0x00, 0x40, 0x26, 0xE5, 0x10, 0x54, 0xFE, 0x12,
-/*10D0*/0x0F, 0x4B, 0x75, 0x83, 0x8E, 0xED, 0xF0, 0xE5,
+/*10D0*/0x0E, 0x98, 0x75, 0x83, 0x8E, 0xED, 0xF0, 0xE5,
0x10, 0x44, 0x01, 0xFD, 0xEB, 0x44, 0x07, 0xF5,
/*10E0*/0x82, 0xED, 0xF0, 0x85, 0x57, 0x82, 0x85, 0x56,
0x83, 0xE0, 0x30, 0xE3, 0x01, 0x09, 0x1E, 0x80,
/*10F0*/0xD4, 0xC2, 0x34, 0xE9, 0xC3, 0x95, 0x54, 0x40,
- 0x02, 0xD2, 0x34, 0x22, 0x22, 0x10, 0x00, 0x00,
+ 0x02, 0xD2, 0x34, 0x22, 0x02, 0x00, 0x06, 0x22,
/*1100*/0x30, 0x30, 0x11, 0x90, 0x10, 0x00, 0xE4, 0x93,
0xF5, 0x10, 0x90, 0x10, 0x10, 0xE4, 0x93, 0xF5,
/*1110*/0x10, 0x12, 0x10, 0x90, 0x12, 0x11, 0x50, 0x22,
- 0xE5, 0xE2, 0x30, 0xE7, 0x0E, 0x12, 0x10, 0x03,
-/*1120*/0xC2, 0x30, 0x30, 0x30, 0x03, 0x12, 0x10, 0xFC,
- 0x20, 0x33, 0x21, 0xE5, 0xFE, 0xC3, 0x94, 0x20,
-/*1130*/0x50, 0x09, 0x53, 0xF9, 0xF7, 0x75, 0xFE, 0x30,
- 0x43, 0xF9, 0x08, 0xE5, 0xF2, 0x30, 0xE7, 0x03,
-/*1140*/0x53, 0xF9, 0x7F, 0xE5, 0xF1, 0x54, 0x70, 0xD3,
- 0x94, 0x00, 0x50, 0xDF, 0x22, 0x00, 0x00, 0x00,
+ 0xE4, 0xFC, 0xC3, 0xED, 0x9F, 0xFA, 0xEF, 0xF5,
+/*1120*/0x83, 0x75, 0x82, 0x00, 0x79, 0xFF, 0xE4, 0x93,
+ 0xCC, 0x6C, 0xCC, 0xA3, 0xD9, 0xF8, 0xDA, 0xF6,
+/*1130*/0xE5, 0xE2, 0x30, 0xE4, 0x02, 0x8C, 0xE5, 0xED,
+ 0x24, 0xFF, 0xFF, 0xEF, 0x75, 0x82, 0xFF, 0xF5,
+/*1140*/0x83, 0xE4, 0x93, 0x6C, 0x70, 0x03, 0x7F, 0x01,
+ 0x22, 0x7F, 0x00, 0x22, 0x22, 0x11, 0x00, 0x00,
/*1150*/0x22, 0x8E, 0x58, 0x8F, 0x59, 0x8C, 0x5A, 0x8D,
0x5B, 0x8A, 0x5C, 0x8B, 0x5D, 0x75, 0x5E, 0x01,
/*1160*/0xE4, 0xF5, 0x5F, 0xF5, 0x60, 0xF5, 0x62, 0x12,
- 0x07, 0x37, 0x75, 0x83, 0xD0, 0xE0, 0xFF, 0xC4,
-/*1170*/0x54, 0x0F, 0xF5, 0x61, 0x12, 0x1E, 0x76, 0x85,
+ 0x07, 0x2A, 0x75, 0x83, 0xD0, 0xE0, 0xFF, 0xC4,
+/*1170*/0x54, 0x0F, 0xF5, 0x61, 0x12, 0x1E, 0xA5, 0x85,
0x59, 0x5E, 0xD3, 0xE5, 0x5E, 0x95, 0x5B, 0xE5,
-/*1180*/0x5A, 0x12, 0x07, 0x78, 0x50, 0x4B, 0x12, 0x07,
- 0x10, 0x75, 0x83, 0xBC, 0xE0, 0x45, 0x5E, 0x12,
-/*1190*/0x07, 0x36, 0x75, 0x83, 0xBE, 0xE0, 0x45, 0x5E,
- 0x12, 0x07, 0x36, 0x75, 0x83, 0xC0, 0xE0, 0x45,
+/*1180*/0x5A, 0x12, 0x07, 0x6B, 0x50, 0x4B, 0x12, 0x07,
+ 0x03, 0x75, 0x83, 0xBC, 0xE0, 0x45, 0x5E, 0x12,
+/*1190*/0x07, 0x29, 0x75, 0x83, 0xBE, 0xE0, 0x45, 0x5E,
+ 0x12, 0x07, 0x29, 0x75, 0x83, 0xC0, 0xE0, 0x45,
/*11A0*/0x5E, 0xF0, 0xAF, 0x5F, 0xE5, 0x60, 0x12, 0x08,
- 0x7B, 0x12, 0x0A, 0x36, 0xAF, 0x62, 0x7E, 0x00,
-/*11B0*/0xAD, 0x5D, 0xAC, 0x5C, 0x12, 0x04, 0x49, 0xE5,
+ 0x78, 0x12, 0x0A, 0xFF, 0xAF, 0x62, 0x7E, 0x00,
+/*11B0*/0xAD, 0x5D, 0xAC, 0x5C, 0x12, 0x04, 0x44, 0xE5,
0x61, 0xAF, 0x5E, 0x7E, 0x00, 0xB4, 0x03, 0x05,
-/*11C0*/0x12, 0x1D, 0xF2, 0x80, 0x07, 0xAD, 0x5D, 0xAC,
+/*11C0*/0x12, 0x1E, 0x21, 0x80, 0x07, 0xAD, 0x5D, 0xAC,
0x5C, 0x12, 0x13, 0x17, 0x05, 0x5E, 0x02, 0x11,
-/*11D0*/0x7A, 0x12, 0x07, 0x10, 0x75, 0x83, 0xBC, 0xE0,
- 0x45, 0x40, 0x12, 0x07, 0x36, 0x75, 0x83, 0xBE,
-/*11E0*/0xE0, 0x45, 0x40, 0x12, 0x07, 0x36, 0x75, 0x83,
+/*11D0*/0x7A, 0x12, 0x07, 0x03, 0x75, 0x83, 0xBC, 0xE0,
+ 0x45, 0x40, 0x12, 0x07, 0x29, 0x75, 0x83, 0xBE,
+/*11E0*/0xE0, 0x45, 0x40, 0x12, 0x07, 0x29, 0x75, 0x83,
0xC0, 0xE0, 0x45, 0x40, 0xF0, 0x22, 0x8E, 0x58,
/*11F0*/0x8F, 0x59, 0x75, 0x5A, 0x01, 0x79, 0x01, 0x75,
- 0x5B, 0x01, 0xE4, 0xFB, 0x12, 0x07, 0x37, 0x75,
+ 0x5B, 0x01, 0xE4, 0xFB, 0x12, 0x07, 0x2A, 0x75,
/*1200*/0x83, 0xAE, 0xE0, 0x54, 0x1A, 0xFF, 0x12, 0x08,
- 0x68, 0xE0, 0xC4, 0x13, 0x54, 0x07, 0xFE, 0xEF,
+ 0x65, 0xE0, 0xC4, 0x13, 0x54, 0x07, 0xFE, 0xEF,
/*1210*/0x70, 0x0C, 0xEE, 0x65, 0x35, 0x70, 0x07, 0x90,
0x07, 0x2F, 0xE0, 0xB4, 0x01, 0x0D, 0xAF, 0x35,
-/*1220*/0x7E, 0x00, 0x12, 0x0E, 0x05, 0xCF, 0xEB, 0xCF,
- 0x02, 0x1E, 0x31, 0xE5, 0x59, 0x64, 0x02, 0x45,
+/*1220*/0x7E, 0x00, 0x12, 0x0E, 0xA9, 0xCF, 0xEB, 0xCF,
+ 0x02, 0x1E, 0x60, 0xE5, 0x59, 0x64, 0x02, 0x45,
/*1230*/0x58, 0x70, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F,
0x00, 0xE5, 0x59, 0x45, 0x58, 0x70, 0x04, 0x7E,
/*1240*/0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F, 0x60,
0x23, 0x85, 0x41, 0x49, 0x85, 0x40, 0x4B, 0xE5,
/*1250*/0x59, 0x45, 0x58, 0x70, 0x2C, 0xAF, 0x5A, 0xFE,
0xCD, 0xE9, 0xCD, 0xFC, 0xAB, 0x59, 0xAA, 0x58,
-/*1260*/0x12, 0x0A, 0x36, 0xAF, 0x5B, 0x7E, 0x00, 0x12,
- 0x1E, 0x31, 0x80, 0x15, 0xAF, 0x5B, 0x7E, 0x00,
-/*1270*/0x12, 0x1E, 0x31, 0x90, 0x07, 0x26, 0x12, 0x07,
- 0x42, 0xE5, 0x49, 0x12, 0x07, 0x3E, 0xE5, 0x4B,
+/*1260*/0x12, 0x0A, 0xFF, 0xAF, 0x5B, 0x7E, 0x00, 0x12,
+ 0x1E, 0x60, 0x80, 0x15, 0xAF, 0x5B, 0x7E, 0x00,
+/*1270*/0x12, 0x1E, 0x60, 0x90, 0x07, 0x26, 0x12, 0x07,
+ 0x35, 0xE5, 0x49, 0x12, 0x07, 0x31, 0xE5, 0x4B,
/*1280*/0xF0, 0xE4, 0xFD, 0xAF, 0x35, 0xFE, 0xFC, 0x12,
- 0x09, 0x22, 0x22, 0x8C, 0x64, 0x8D, 0x65, 0x12,
-/*1290*/0x08, 0xE7, 0x40, 0x3C, 0xE5, 0x65, 0x45, 0x64,
- 0x70, 0x10, 0x12, 0x09, 0x11, 0xC3, 0xE5, 0x3E,
-/*12A0*/0x12, 0x07, 0x76, 0x40, 0x3B, 0x12, 0x08, 0xA2,
+ 0x09, 0x15, 0x22, 0x8C, 0x64, 0x8D, 0x65, 0x12,
+/*1290*/0x08, 0xDA, 0x40, 0x3C, 0xE5, 0x65, 0x45, 0x64,
+ 0x70, 0x10, 0x12, 0x09, 0x04, 0xC3, 0xE5, 0x3E,
+/*12A0*/0x12, 0x07, 0x69, 0x40, 0x3B, 0x12, 0x08, 0x95,
0x80, 0x18, 0xE5, 0x3E, 0xC3, 0x95, 0x38, 0x40,
/*12B0*/0x1D, 0x85, 0x3E, 0x38, 0xE5, 0x3E, 0x60, 0x05,
0x85, 0x3F, 0x39, 0x80, 0x03, 0x85, 0x39, 0x39,
-/*12C0*/0x8F, 0x3A, 0x12, 0x07, 0xB5, 0xE5, 0x3E, 0x12,
- 0x07, 0x60, 0xE5, 0x3F, 0xF0, 0x22, 0x80, 0x3B,
+/*12C0*/0x8F, 0x3A, 0x12, 0x07, 0xA8, 0xE5, 0x3E, 0x12,
+ 0x07, 0x53, 0xE5, 0x3F, 0xF0, 0x22, 0x80, 0x3B,
/*12D0*/0xE5, 0x65, 0x45, 0x64, 0x70, 0x11, 0x12, 0x07,
- 0x6C, 0x40, 0x05, 0x12, 0x08, 0xAB, 0x80, 0x1F,
-/*12E0*/0x12, 0x07, 0x4B, 0xE5, 0x41, 0xF0, 0x22, 0xE5,
+ 0x5F, 0x40, 0x05, 0x12, 0x08, 0x9E, 0x80, 0x1F,
+/*12E0*/0x12, 0x07, 0x3E, 0xE5, 0x41, 0xF0, 0x22, 0xE5,
0x3C, 0xC3, 0x95, 0x38, 0x40, 0x1D, 0x85, 0x3C,
/*12F0*/0x38, 0xE5, 0x3C, 0x60, 0x05, 0x85, 0x3D, 0x39,
0x80, 0x03, 0x85, 0x39, 0x39, 0x8F, 0x3A, 0x12,
-/*1300*/0x07, 0xB5, 0xE5, 0x3C, 0x12, 0x07, 0x60, 0xE5,
- 0x3D, 0xF0, 0x22, 0x12, 0x07, 0xAC, 0xE5, 0x38,
-/*1310*/0x12, 0x07, 0x60, 0xE5, 0x39, 0xF0, 0x22, 0x8C,
- 0x63, 0x8D, 0x64, 0x12, 0x08, 0xE7, 0x40, 0x3C,
+/*1300*/0x07, 0xA8, 0xE5, 0x3C, 0x12, 0x07, 0x53, 0xE5,
+ 0x3D, 0xF0, 0x22, 0x12, 0x07, 0x9F, 0xE5, 0x38,
+/*1310*/0x12, 0x07, 0x53, 0xE5, 0x39, 0xF0, 0x22, 0x8C,
+ 0x63, 0x8D, 0x64, 0x12, 0x08, 0xDA, 0x40, 0x3C,
/*1320*/0xE5, 0x64, 0x45, 0x63, 0x70, 0x10, 0x12, 0x09,
- 0x11, 0xC3, 0xE5, 0x3E, 0x12, 0x07, 0x76, 0x40,
-/*1330*/0x3B, 0x12, 0x08, 0xA2, 0x80, 0x18, 0xE5, 0x3E,
+ 0x04, 0xC3, 0xE5, 0x3E, 0x12, 0x07, 0x69, 0x40,
+/*1330*/0x3B, 0x12, 0x08, 0x95, 0x80, 0x18, 0xE5, 0x3E,
0xC3, 0x95, 0x38, 0x40, 0x1D, 0x85, 0x3E, 0x38,
/*1340*/0xE5, 0x3E, 0x60, 0x05, 0x85, 0x3F, 0x39, 0x80,
0x03, 0x85, 0x39, 0x39, 0x8F, 0x3A, 0x12, 0x07,
-/*1350*/0xB5, 0xE5, 0x3E, 0x12, 0x07, 0x60, 0xE5, 0x3F,
+/*1350*/0xA8, 0xE5, 0x3E, 0x12, 0x07, 0x53, 0xE5, 0x3F,
0xF0, 0x22, 0x80, 0x3B, 0xE5, 0x64, 0x45, 0x63,
-/*1360*/0x70, 0x11, 0x12, 0x07, 0x6C, 0x40, 0x05, 0x12,
- 0x08, 0xAB, 0x80, 0x1F, 0x12, 0x07, 0x4B, 0xE5,
+/*1360*/0x70, 0x11, 0x12, 0x07, 0x5F, 0x40, 0x05, 0x12,
+ 0x08, 0x9E, 0x80, 0x1F, 0x12, 0x07, 0x3E, 0xE5,
/*1370*/0x41, 0xF0, 0x22, 0xE5, 0x3C, 0xC3, 0x95, 0x38,
0x40, 0x1D, 0x85, 0x3C, 0x38, 0xE5, 0x3C, 0x60,
/*1380*/0x05, 0x85, 0x3D, 0x39, 0x80, 0x03, 0x85, 0x39,
- 0x39, 0x8F, 0x3A, 0x12, 0x07, 0xB5, 0xE5, 0x3C,
-/*1390*/0x12, 0x07, 0x60, 0xE5, 0x3D, 0xF0, 0x22, 0x12,
- 0x07, 0xAC, 0xE5, 0x38, 0x12, 0x07, 0x60, 0xE5,
+ 0x39, 0x8F, 0x3A, 0x12, 0x07, 0xA8, 0xE5, 0x3C,
+/*1390*/0x12, 0x07, 0x53, 0xE5, 0x3D, 0xF0, 0x22, 0x12,
+ 0x07, 0x9F, 0xE5, 0x38, 0x12, 0x07, 0x53, 0xE5,
/*13A0*/0x39, 0xF0, 0x22, 0xE5, 0x0D, 0xFE, 0xE5, 0x08,
0x8E, 0x54, 0x44, 0x05, 0xF5, 0x55, 0x75, 0x15,
-/*13B0*/0x0F, 0xF5, 0x82, 0x12, 0x0F, 0x2D, 0x12, 0x17,
- 0x25, 0x20, 0x31, 0x05, 0x75, 0x15, 0x03, 0x80,
+/*13B0*/0x0F, 0xF5, 0x82, 0x12, 0x0E, 0x7A, 0x12, 0x17,
+ 0xA3, 0x20, 0x31, 0x05, 0x75, 0x15, 0x03, 0x80,
/*13C0*/0x03, 0x75, 0x15, 0x0B, 0xE5, 0x0A, 0xC3, 0x94,
0x01, 0x50, 0x38, 0x12, 0x14, 0x20, 0x20, 0x31,
/*13D0*/0x06, 0x05, 0x15, 0x05, 0x15, 0x80, 0x04, 0x15,
0x15, 0x15, 0x15, 0xE5, 0x0A, 0xC3, 0x94, 0x01,
/*13E0*/0x50, 0x21, 0x12, 0x14, 0x20, 0x20, 0x31, 0x04,
0x05, 0x15, 0x80, 0x02, 0x15, 0x15, 0xE5, 0x0A,
-/*13F0*/0xC3, 0x94, 0x01, 0x50, 0x0E, 0x12, 0x0F, 0x2A,
- 0x12, 0x17, 0x25, 0x20, 0x31, 0x05, 0x05, 0x15,
-/*1400*/0x12, 0x0F, 0x2A, 0xE5, 0x15, 0xB4, 0x08, 0x04,
+/*13F0*/0xC3, 0x94, 0x01, 0x50, 0x0E, 0x12, 0x0E, 0x77,
+ 0x12, 0x17, 0xA3, 0x20, 0x31, 0x05, 0x05, 0x15,
+/*1400*/0x12, 0x0E, 0x77, 0xE5, 0x15, 0xB4, 0x08, 0x04,
0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x15,
/*1410*/0xB4, 0x07, 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E,
0x00, 0xEE, 0x4F, 0x60, 0x02, 0x05, 0x7F, 0x22,
/*1420*/0x85, 0x55, 0x82, 0x85, 0x54, 0x83, 0xE5, 0x15,
- 0xF0, 0x12, 0x17, 0x25, 0x22, 0x12, 0x07, 0x37,
-/*1430*/0x75, 0x83, 0xAE, 0x74, 0xFF, 0x12, 0x07, 0x36,
+ 0xF0, 0x12, 0x17, 0xA3, 0x22, 0x12, 0x07, 0x2A,
+/*1430*/0x75, 0x83, 0xAE, 0x74, 0xFF, 0x12, 0x07, 0x29,
0xE0, 0x54, 0x1A, 0xF5, 0x34, 0xE0, 0xC4, 0x13,
/*1440*/0x54, 0x07, 0xF5, 0x35, 0x24, 0xFE, 0x60, 0x24,
0x24, 0xFE, 0x60, 0x3C, 0x24, 0x04, 0x70, 0x63,
/*1450*/0x75, 0x31, 0x2D, 0xE5, 0x08, 0xFD, 0x74, 0xB6,
- 0x12, 0x07, 0x9F, 0x74, 0xBC, 0x90, 0x07, 0x22,
-/*1460*/0x12, 0x07, 0xA2, 0x74, 0x90, 0x12, 0x07, 0xC0,
+ 0x12, 0x07, 0x92, 0x74, 0xBC, 0x90, 0x07, 0x22,
+/*1460*/0x12, 0x07, 0x95, 0x74, 0x90, 0x12, 0x07, 0xB3,
0x74, 0x92, 0x80, 0x3C, 0x75, 0x31, 0x3A, 0xE5,
-/*1470*/0x08, 0xFD, 0x74, 0xBA, 0x12, 0x07, 0x9F, 0x74,
- 0xC0, 0x90, 0x07, 0x22, 0x12, 0x07, 0xC3, 0x74,
-/*1480*/0xC4, 0x12, 0x07, 0xC0, 0x74, 0xC8, 0x80, 0x20,
+/*1470*/0x08, 0xFD, 0x74, 0xBA, 0x12, 0x07, 0x92, 0x74,
+ 0xC0, 0x90, 0x07, 0x22, 0x12, 0x07, 0xB6, 0x74,
+/*1480*/0xC4, 0x12, 0x07, 0xB3, 0x74, 0xC8, 0x80, 0x20,
0x75, 0x31, 0x35, 0xE5, 0x08, 0xFD, 0x74, 0xB8,
-/*1490*/0x12, 0x07, 0x9F, 0x74, 0xBE, 0xFF, 0xED, 0x44,
+/*1490*/0x12, 0x07, 0x92, 0x74, 0xBE, 0xFF, 0xED, 0x44,
0x07, 0x90, 0x07, 0x22, 0xCF, 0xF0, 0xA3, 0xEF,
-/*14A0*/0xF0, 0x74, 0xC2, 0x12, 0x07, 0xC0, 0x74, 0xC6,
+/*14A0*/0xF0, 0x74, 0xC2, 0x12, 0x07, 0xB3, 0x74, 0xC6,
0xFF, 0xED, 0x44, 0x07, 0xA3, 0xCF, 0xF0, 0xA3,
/*14B0*/0xEF, 0xF0, 0x22, 0x75, 0x34, 0x01, 0x22, 0x8E,
0x58, 0x8F, 0x59, 0x8C, 0x5A, 0x8D, 0x5B, 0x8A,
/*14C0*/0x5C, 0x8B, 0x5D, 0x75, 0x5E, 0x01, 0xE4, 0xF5,
- 0x5F, 0x12, 0x1E, 0x76, 0x85, 0x59, 0x5E, 0xD3,
+ 0x5F, 0x12, 0x1E, 0xA5, 0x85, 0x59, 0x5E, 0xD3,
/*14D0*/0xE5, 0x5E, 0x95, 0x5B, 0xE5, 0x5A, 0x12, 0x07,
- 0x78, 0x50, 0x57, 0xE5, 0x5D, 0x45, 0x5C, 0x70,
-/*14E0*/0x30, 0x12, 0x07, 0x37, 0x75, 0x83, 0x92, 0xE5,
- 0x5E, 0x12, 0x07, 0x36, 0x75, 0x83, 0xC6, 0xE5,
-/*14F0*/0x5E, 0x12, 0x07, 0x36, 0x75, 0x83, 0xC8, 0xE5,
- 0x5E, 0x12, 0x07, 0x36, 0x75, 0x83, 0x90, 0xE5,
-/*1500*/0x5E, 0x12, 0x07, 0x36, 0x75, 0x83, 0xC2, 0xE5,
- 0x5E, 0x12, 0x07, 0x36, 0x75, 0x83, 0xC4, 0x80,
-/*1510*/0x03, 0x12, 0x07, 0x3F, 0xE5, 0x5E, 0xF0, 0xAF,
+ 0x6B, 0x50, 0x57, 0xE5, 0x5D, 0x45, 0x5C, 0x70,
+/*14E0*/0x30, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x92, 0xE5,
+ 0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC6, 0xE5,
+/*14F0*/0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC8, 0xE5,
+ 0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0x90, 0xE5,
+/*1500*/0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2, 0xE5,
+ 0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0x80,
+/*1510*/0x03, 0x12, 0x07, 0x32, 0xE5, 0x5E, 0xF0, 0xAF,
0x5F, 0x7E, 0x00, 0xAD, 0x5D, 0xAC, 0x5C, 0x12,
-/*1520*/0x04, 0x49, 0xAF, 0x5E, 0x7E, 0x00, 0xAD, 0x5D,
- 0xAC, 0x5C, 0x12, 0x0B, 0xD9, 0x05, 0x5E, 0x02,
+/*1520*/0x04, 0x44, 0xAF, 0x5E, 0x7E, 0x00, 0xAD, 0x5D,
+ 0xAC, 0x5C, 0x12, 0x0B, 0xD1, 0x05, 0x5E, 0x02,
/*1530*/0x14, 0xCF, 0xAB, 0x5D, 0xAA, 0x5C, 0xAD, 0x5B,
0xAC, 0x5A, 0xAF, 0x59, 0xAE, 0x58, 0x02, 0x1B,
-/*1540*/0xD4, 0x8C, 0x5C, 0x8D, 0x5D, 0x8A, 0x5E, 0x8B,
+/*1540*/0xFB, 0x8C, 0x5C, 0x8D, 0x5D, 0x8A, 0x5E, 0x8B,
0x5F, 0x75, 0x60, 0x01, 0xE4, 0xF5, 0x61, 0xF5,
-/*1550*/0x62, 0xF5, 0x63, 0x12, 0x1E, 0x76, 0x8F, 0x60,
+/*1550*/0x62, 0xF5, 0x63, 0x12, 0x1E, 0xA5, 0x8F, 0x60,
0xD3, 0xE5, 0x60, 0x95, 0x5D, 0xE5, 0x5C, 0x12,
-/*1560*/0x07, 0x78, 0x50, 0x61, 0xE5, 0x5F, 0x45, 0x5E,
- 0x70, 0x27, 0x12, 0x07, 0x37, 0x75, 0x83, 0xB6,
-/*1570*/0xE5, 0x60, 0x12, 0x07, 0x36, 0x75, 0x83, 0xB8,
- 0xE5, 0x60, 0x12, 0x07, 0x36, 0x75, 0x83, 0xBA,
+/*1560*/0x07, 0x6B, 0x50, 0x61, 0xE5, 0x5F, 0x45, 0x5E,
+ 0x70, 0x27, 0x12, 0x07, 0x2A, 0x75, 0x83, 0xB6,
+/*1570*/0xE5, 0x60, 0x12, 0x07, 0x29, 0x75, 0x83, 0xB8,
+ 0xE5, 0x60, 0x12, 0x07, 0x29, 0x75, 0x83, 0xBA,
/*1580*/0xE5, 0x60, 0xF0, 0xAF, 0x61, 0x7E, 0x00, 0xE5,
- 0x62, 0x12, 0x08, 0x7D, 0x12, 0x0A, 0x36, 0x80,
-/*1590*/0x19, 0x90, 0x07, 0x24, 0x12, 0x07, 0x42, 0xE5,
- 0x60, 0x12, 0x07, 0x36, 0x75, 0x83, 0x8E, 0xE4,
-/*15A0*/0x12, 0x07, 0x36, 0x74, 0x01, 0x12, 0x07, 0x36,
+ 0x62, 0x12, 0x08, 0x7A, 0x12, 0x0A, 0xFF, 0x80,
+/*1590*/0x19, 0x90, 0x07, 0x24, 0x12, 0x07, 0x35, 0xE5,
+ 0x60, 0x12, 0x07, 0x29, 0x75, 0x83, 0x8E, 0xE4,
+/*15A0*/0x12, 0x07, 0x29, 0x74, 0x01, 0x12, 0x07, 0x29,
0xE4, 0xF0, 0xAF, 0x63, 0x7E, 0x00, 0xAD, 0x5F,
-/*15B0*/0xAC, 0x5E, 0x12, 0x04, 0x49, 0xAF, 0x60, 0x7E,
+/*15B0*/0xAC, 0x5E, 0x12, 0x04, 0x44, 0xAF, 0x60, 0x7E,
0x00, 0xAD, 0x5F, 0xAC, 0x5E, 0x12, 0x12, 0x8B,
-/*15C0*/0x05, 0x60, 0x02, 0x15, 0x58, 0x22, 0xE4, 0xF5,
- 0x58, 0xF5, 0x59, 0xAF, 0x08, 0xEF, 0x44, 0x07,
-/*15D0*/0xF5, 0x82, 0x75, 0x83, 0xD0, 0xE0, 0xFD, 0xC4,
- 0x54, 0x0F, 0xF5, 0x5A, 0xEF, 0x44, 0x07, 0xF5,
-/*15E0*/0x82, 0x75, 0x83, 0x80, 0x74, 0x01, 0xF0, 0x12,
- 0x08, 0x9A, 0x75, 0x83, 0x82, 0xE5, 0x45, 0xF0,
-/*15F0*/0xEF, 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0x8A,
- 0x74, 0xFF, 0xF0, 0x12, 0x19, 0x74, 0x12, 0x07,
-/*1600*/0x37, 0x75, 0x83, 0xBC, 0xE0, 0x54, 0xEF, 0x12,
- 0x07, 0x36, 0x75, 0x83, 0xBE, 0xE0, 0x54, 0xEF,
-/*1610*/0x12, 0x07, 0x36, 0x75, 0x83, 0xC0, 0xE0, 0x54,
- 0xEF, 0x12, 0x07, 0x36, 0x75, 0x83, 0xBC, 0xE0,
-/*1620*/0x44, 0x10, 0x12, 0x07, 0x36, 0x75, 0x83, 0xBE,
- 0xE0, 0x44, 0x10, 0x12, 0x07, 0x36, 0x75, 0x83,
-/*1630*/0xC0, 0xE0, 0x44, 0x10, 0xF0, 0xAF, 0x58, 0xE5,
- 0x59, 0x12, 0x08, 0x7B, 0x02, 0x0A, 0x36, 0xE4,
-/*1640*/0xF5, 0x58, 0x7D, 0x01, 0xF5, 0x59, 0xAF, 0x35,
- 0xFE, 0xFC, 0x12, 0x09, 0x22, 0x12, 0x07, 0x37,
-/*1650*/0x75, 0x83, 0xB6, 0x74, 0x10, 0x12, 0x07, 0x36,
- 0x75, 0x83, 0xB8, 0x74, 0x10, 0x12, 0x07, 0x36,
-/*1660*/0x75, 0x83, 0xBA, 0x74, 0x10, 0x12, 0x07, 0x36,
- 0x75, 0x83, 0xBC, 0x74, 0x10, 0x12, 0x07, 0x36,
-/*1670*/0x75, 0x83, 0xBE, 0x74, 0x10, 0x12, 0x07, 0x36,
- 0x75, 0x83, 0xC0, 0x74, 0x10, 0x12, 0x07, 0x36,
-/*1680*/0x75, 0x83, 0x90, 0xE4, 0x12, 0x07, 0x36, 0x75,
- 0x83, 0xC2, 0xE4, 0x12, 0x07, 0x36, 0x75, 0x83,
-/*1690*/0xC4, 0xE4, 0x12, 0x07, 0x36, 0x75, 0x83, 0x92,
- 0xE4, 0x12, 0x07, 0x36, 0x75, 0x83, 0xC6, 0xE4,
-/*16A0*/0x12, 0x07, 0x36, 0x75, 0x83, 0xC8, 0xE4, 0xF0,
- 0xAF, 0x58, 0xFE, 0xE5, 0x59, 0x12, 0x08, 0x7D,
-/*16B0*/0x02, 0x0A, 0x36, 0xE5, 0xE2, 0x30, 0xE4, 0x6C,
- 0xE5, 0xE7, 0x54, 0xC0, 0x64, 0x40, 0x70, 0x64,
-/*16C0*/0xE5, 0x09, 0xC4, 0x54, 0x30, 0xFE, 0xE5, 0x08,
- 0x25, 0xE0, 0x25, 0xE0, 0x54, 0xC0, 0x4E, 0xFE,
-/*16D0*/0xEF, 0x54, 0x3F, 0x4E, 0xFD, 0xE5, 0x2B, 0xAE,
- 0x2A, 0x78, 0x02, 0xC3, 0x33, 0xCE, 0x33, 0xCE,
-/*16E0*/0xD8, 0xF9, 0xF5, 0x82, 0x8E, 0x83, 0xED, 0xF0,
- 0xE5, 0x2B, 0xAE, 0x2A, 0x78, 0x02, 0xC3, 0x33,
-/*16F0*/0xCE, 0x33, 0xCE, 0xD8, 0xF9, 0xFF, 0xF5, 0x82,
- 0x8E, 0x83, 0xA3, 0xE5, 0xFE, 0xF0, 0x8F, 0x82,
-/*1700*/0x8E, 0x83, 0xA3, 0xA3, 0xE5, 0xFD, 0xF0, 0x8F,
- 0x82, 0x8E, 0x83, 0xA3, 0xA3, 0xA3, 0xE5, 0xFC,
-/*1710*/0xF0, 0xC3, 0xE5, 0x2B, 0x94, 0xFA, 0xE5, 0x2A,
- 0x94, 0x00, 0x50, 0x08, 0x05, 0x2B, 0xE5, 0x2B,
-/*1720*/0x70, 0x02, 0x05, 0x2A, 0x22, 0xE4, 0xFF, 0xE4,
- 0xF5, 0x58, 0xF5, 0x56, 0xF5, 0x57, 0x74, 0x82,
-/*1730*/0xFC, 0x12, 0x0E, 0xB7, 0x8C, 0x83, 0xE0, 0xF5,
- 0x10, 0x54, 0x7F, 0xF0, 0xE5, 0x10, 0x44, 0x80,
-/*1740*/0x12, 0x0F, 0x4B, 0xED, 0xF0, 0x7E, 0x0A, 0x12,
- 0x0E, 0xB7, 0x75, 0x83, 0xA0, 0xE0, 0x20, 0xE0,
-/*1750*/0x26, 0xDE, 0xF4, 0x05, 0x57, 0xE5, 0x57, 0x70,
- 0x02, 0x05, 0x56, 0xE5, 0x14, 0x24, 0x01, 0xFD,
-/*1760*/0xE4, 0x33, 0xFC, 0xD3, 0xE5, 0x57, 0x9D, 0xE5,
- 0x56, 0x9C, 0x40, 0xD9, 0xE5, 0x0A, 0x94, 0x20,
-/*1770*/0x50, 0x02, 0x05, 0x0A, 0x43, 0xE1, 0x08, 0xC2,
- 0x31, 0x12, 0x0E, 0xB7, 0x75, 0x83, 0xA6, 0xE0,
-/*1780*/0x55, 0x12, 0x65, 0x12, 0x70, 0x03, 0xD2, 0x31,
- 0x22, 0xC2, 0x31, 0x22, 0x90, 0x07, 0x26, 0xE0,
-/*1790*/0xFA, 0xA3, 0xE0, 0xF5, 0x82, 0x8A, 0x83, 0xE0,
- 0xF5, 0x41, 0xE5, 0x39, 0xC3, 0x95, 0x41, 0x40,
-/*17A0*/0x26, 0xE5, 0x39, 0x95, 0x41, 0xC3, 0x9F, 0xEE,
- 0x12, 0x07, 0x78, 0x40, 0x04, 0x7C, 0x01, 0x80,
-/*17B0*/0x02, 0x7C, 0x00, 0xE5, 0x41, 0x64, 0x3F, 0x60,
- 0x04, 0x7B, 0x01, 0x80, 0x02, 0x7B, 0x00, 0xEC,
-/*17C0*/0x5B, 0x60, 0x29, 0x05, 0x41, 0x80, 0x28, 0xC3,
- 0xE5, 0x41, 0x95, 0x39, 0xC3, 0x9F, 0xEE, 0x12,
-/*17D0*/0x07, 0x78, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02,
- 0x7F, 0x00, 0xE5, 0x41, 0x60, 0x04, 0x7E, 0x01,
-/*17E0*/0x80, 0x02, 0x7E, 0x00, 0xEF, 0x5E, 0x60, 0x04,
- 0x15, 0x41, 0x80, 0x03, 0x85, 0x39, 0x41, 0x85,
-/*17F0*/0x3A, 0x40, 0x22, 0xE5, 0xE2, 0x30, 0xE4, 0x60,
- 0xE5, 0xE1, 0x30, 0xE2, 0x5B, 0xE5, 0x09, 0x70,
-/*1800*/0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5,
- 0x08, 0x70, 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E,
-/*1810*/0x00, 0xEE, 0x5F, 0x60, 0x43, 0x53, 0xF9, 0xF8,
- 0xE5, 0xE2, 0x30, 0xE4, 0x3B, 0xE5, 0xE1, 0x30,
-/*1820*/0xE2, 0x2E, 0x43, 0xFA, 0x02, 0x53, 0xFA, 0xFB,
- 0xE4, 0xF5, 0x10, 0x90, 0x94, 0x70, 0xE5, 0x10,
-/*1830*/0xF0, 0xE5, 0xE1, 0x30, 0xE2, 0xE7, 0x90, 0x94,
- 0x70, 0xE0, 0x65, 0x10, 0x60, 0x03, 0x43, 0xFA,
-/*1840*/0x04, 0x05, 0x10, 0x90, 0x94, 0x70, 0xE5, 0x10,
- 0xF0, 0x70, 0xE6, 0x12, 0x00, 0x06, 0x80, 0xE1,
-/*1850*/0x53, 0xFA, 0xFD, 0x53, 0xFA, 0xFB, 0x80, 0xC0,
- 0x22, 0x8F, 0x54, 0x12, 0x00, 0x06, 0xE5, 0xE1,
-/*1860*/0x30, 0xE0, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F,
- 0x00, 0xE5, 0x7E, 0xD3, 0x94, 0x05, 0x40, 0x04,
-/*1870*/0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F,
- 0x60, 0x3D, 0x85, 0x54, 0x11, 0xE5, 0xE2, 0x20,
-/*1880*/0xE1, 0x32, 0x74, 0xCE, 0x12, 0x1B, 0x95, 0x30,
- 0xE7, 0x04, 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00,
-/*1890*/0x8F, 0x82, 0x8E, 0x83, 0xE0, 0x30, 0xE6, 0x04,
- 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xEF, 0x5D,
-/*18A0*/0x70, 0x15, 0x12, 0x19, 0xCF, 0x74, 0xCE, 0x12,
- 0x1B, 0x95, 0x30, 0xE6, 0x07, 0xE0, 0x44, 0x80,
-/*18B0*/0xF0, 0x43, 0xF9, 0x80, 0x12, 0x17, 0xF3, 0x22,
- 0x12, 0x0E, 0xF7, 0xE5, 0x16, 0x25, 0xE0, 0x25,
-/*18C0*/0xE0, 0x24, 0x89, 0xF5, 0x82, 0xE4, 0x34, 0x1A,
- 0xF5, 0x83, 0xE4, 0x93, 0xF5, 0x0F, 0xE5, 0x16,
-/*18D0*/0x25, 0xE0, 0x25, 0xE0, 0x24, 0x8A, 0xF5, 0x82,
- 0xE4, 0x34, 0x1A, 0xF5, 0x83, 0xE4, 0x93, 0xF5,
-/*18E0*/0x0E, 0x12, 0x0F, 0x18, 0xF5, 0x10, 0xE5, 0x0F,
- 0x54, 0xF0, 0x12, 0x0E, 0xCA, 0x75, 0x83, 0x8C,
-/*18F0*/0xEF, 0xF0, 0xE5, 0x0F, 0x30, 0xE0, 0x0C, 0x12,
- 0x0E, 0xB7, 0x75, 0x83, 0x86, 0xE0, 0x44, 0x40,
-/*1900*/0xF0, 0x80, 0x0A, 0x12, 0x0E, 0xB7, 0x75, 0x83,
- 0x86, 0xE0, 0x54, 0xBF, 0xF0, 0x12, 0x0F, 0x44,
-/*1910*/0x75, 0x83, 0x82, 0xE5, 0x0E, 0xF0, 0x22, 0x7F,
- 0x05, 0x12, 0x16, 0xB3, 0x12, 0x0E, 0xB7, 0x12,
-/*1920*/0x0E, 0xE6, 0x74, 0x02, 0xF0, 0x74, 0x8E, 0xFE,
- 0x12, 0x0E, 0xB7, 0x12, 0x0E, 0xBE, 0xEF, 0xF0,
-/*1930*/0x75, 0x15, 0x70, 0x12, 0x1E, 0x9A, 0x20, 0x34,
- 0x05, 0x75, 0x15, 0x10, 0x80, 0x03, 0x75, 0x15,
-/*1940*/0x50, 0x12, 0x1E, 0x9A, 0x20, 0x34, 0x04, 0x74,
- 0x10, 0x80, 0x02, 0x74, 0xF0, 0x25, 0x15, 0xF5,
-/*1950*/0x15, 0x12, 0x0E, 0xD4, 0xEF, 0xF0, 0x12, 0x10,
- 0x91, 0x20, 0x34, 0x17, 0xE5, 0x15, 0x64, 0x30,
-/*1960*/0x60, 0x0C, 0x74, 0x10, 0x25, 0x15, 0xF5, 0x15,
- 0xB4, 0x80, 0x03, 0xE4, 0xF5, 0x15, 0x12, 0x0E,
-/*1970*/0xD4, 0xEF, 0xF0, 0x22, 0xE4, 0x90, 0x07, 0x2A,
- 0xF0, 0xA3, 0xF0, 0x12, 0x07, 0x37, 0x75, 0x83,
-/*1980*/0x82, 0xE0, 0x54, 0x7F, 0x12, 0x07, 0x36, 0xE0,
- 0x44, 0x80, 0xF0, 0x12, 0x00, 0x3D, 0x12, 0x08,
-/*1990*/0x98, 0x75, 0x83, 0xA0, 0xE0, 0x20, 0xE0, 0x1A,
- 0x90, 0x07, 0x2B, 0xE0, 0x04, 0xF0, 0x70, 0x06,
-/*19A0*/0x90, 0x07, 0x2A, 0xE0, 0x04, 0xF0, 0x90, 0x07,
- 0x2A, 0xE0, 0xB4, 0x10, 0xE1, 0xA3, 0xE0, 0xB4,
-/*19B0*/0x00, 0xDC, 0xEE, 0x44, 0xA6, 0xFC, 0xEF, 0x44,
- 0x07, 0xF5, 0x82, 0x8C, 0x83, 0xE0, 0xF5, 0x32,
-/*19C0*/0xEE, 0x44, 0xA8, 0xFE, 0xEF, 0x44, 0x07, 0xF5,
- 0x82, 0x8E, 0x83, 0xE0, 0xF5, 0x33, 0x22, 0x90,
-/*19D0*/0x10, 0xFD, 0xE4, 0x93, 0x90, 0x07, 0x2E, 0xF0,
- 0x12, 0x14, 0x2D, 0x12, 0x1A, 0x28, 0xE5, 0x34,
-/*19E0*/0x70, 0x45, 0x12, 0x07, 0x37, 0x75, 0x83, 0xCE,
- 0xE0, 0xFF, 0x13, 0x13, 0x13, 0x54, 0x07, 0xF5,
-/*19F0*/0x36, 0x54, 0x0F, 0x24, 0xFE, 0x60, 0x0B, 0x24,
- 0xFE, 0x60, 0x0A, 0x24, 0x03, 0x70, 0x28, 0x02,
-/*1A00*/0x1D, 0x7F, 0x02, 0x1E, 0x62, 0x12, 0x10, 0x14,
- 0xE4, 0xF5, 0x55, 0x12, 0x1D, 0x56, 0x05, 0x55,
-/*1A10*/0xE5, 0x55, 0xC3, 0x94, 0x05, 0x40, 0xF4, 0x12,
- 0x07, 0x37, 0x75, 0x83, 0xCE, 0xE0, 0x54, 0xC7,
-/*1A20*/0x12, 0x07, 0x36, 0xE0, 0x44, 0x08, 0xF0, 0x22,
- 0x12, 0x08, 0x98, 0x75, 0x83, 0xD0, 0xE0, 0xC4,
-/*1A30*/0x54, 0x0F, 0xFD, 0x75, 0x43, 0x01, 0x75, 0x44,
- 0xFF, 0x12, 0x08, 0xB7, 0x74, 0x04, 0xF0, 0x75,
-/*1A40*/0x3B, 0x01, 0xED, 0x14, 0x60, 0x0C, 0x14, 0x60,
- 0x0B, 0x14, 0x60, 0x0F, 0x24, 0x03, 0x70, 0x0B,
-/*1A50*/0x80, 0x09, 0x80, 0x00, 0x12, 0x08, 0xB4, 0x04,
- 0xF0, 0x80, 0x06, 0x12, 0x08, 0xB4, 0x74, 0x04,
-/*1A60*/0xF0, 0xEE, 0x44, 0x82, 0xFE, 0xEF, 0x44, 0x07,
- 0xF5, 0x82, 0x8E, 0x83, 0xE5, 0x45, 0x12, 0x08,
-/*1A70*/0xCB, 0x75, 0x83, 0x82, 0xE5, 0x31, 0x12, 0x07,
- 0x36, 0x75, 0x83, 0x86, 0xE0, 0x44, 0x08, 0xF0,
-/*1A80*/0x22, 0x01, 0x1F, 0x10, 0x00, 0x04, 0x20, 0x00,
- 0x90, 0x00, 0x20, 0x0F, 0x92, 0x00, 0x21, 0x0F,
-/*1A90*/0x94, 0x00, 0x22, 0x0F, 0x96, 0x00, 0x23, 0x0F,
- 0x98, 0x00, 0x24, 0x0F, 0x9A, 0x00, 0x25, 0x0F,
-/*1AA0*/0x9C, 0x00, 0x26, 0x0F, 0x9E, 0x00, 0x27, 0x0F,
- 0xA0, 0x01, 0x20, 0x01, 0xA2, 0x01, 0x21, 0x01,
-/*1AB0*/0xA4, 0x01, 0x22, 0x01, 0xA6, 0x01, 0x23, 0x01,
- 0xA8, 0x01, 0x24, 0x01, 0xAA, 0x01, 0x25, 0x01,
-/*1AC0*/0xAC, 0x01, 0x26, 0x01, 0xAE, 0x01, 0x27, 0x01,
- 0xB0, 0x01, 0x28, 0x01, 0xB4, 0x00, 0x28, 0x0F,
-/*1AD0*/0xB6, 0x40, 0x28, 0x0F, 0xB8, 0x61, 0x28, 0x01,
- 0xCB, 0xEF, 0xCB, 0xCA, 0xEE, 0xCA, 0x7F, 0x01,
-/*1AE0*/0xE4, 0xFD, 0xEB, 0x4A, 0x70, 0x24, 0xE5, 0x08,
- 0xF5, 0x82, 0x74, 0xB6, 0x12, 0x08, 0x2C, 0xE5,
-/*1AF0*/0x08, 0xF5, 0x82, 0x74, 0xB8, 0x12, 0x08, 0x2C,
- 0xE5, 0x08, 0xF5, 0x82, 0x74, 0xBA, 0x12, 0x08,
-/*1B00*/0x2C, 0x7E, 0x00, 0x7C, 0x00, 0x12, 0x0A, 0x36,
- 0x80, 0x12, 0x90, 0x07, 0x26, 0x12, 0x07, 0x42,
-/*1B10*/0xE5, 0x41, 0xF0, 0x90, 0x07, 0x24, 0x12, 0x07,
- 0x42, 0xE5, 0x40, 0xF0, 0x12, 0x07, 0x37, 0x75,
-/*1B20*/0x83, 0x8E, 0xE4, 0x12, 0x07, 0x36, 0x74, 0x01,
- 0x12, 0x07, 0x36, 0xE4, 0xF0, 0x22, 0xE4, 0xF5,
-/*1B30*/0x26, 0xF5, 0x27, 0x53, 0xE1, 0xFE, 0xF5, 0x2A,
- 0x75, 0x2B, 0x01, 0xF5, 0x08, 0x7F, 0x01, 0x12,
-/*1B40*/0x16, 0xB3, 0x30, 0x30, 0x1C, 0x90, 0x1A, 0x82,
- 0xE4, 0x93, 0xF5, 0x10, 0x90, 0x1F, 0xF9, 0xE4,
-/*1B50*/0x93, 0xF5, 0x10, 0x90, 0x00, 0x41, 0xE4, 0x93,
- 0xF5, 0x10, 0x90, 0x0F, 0xF9, 0xE4, 0x93, 0xF5,
-/*1B60*/0x10, 0x7F, 0x02, 0x12, 0x16, 0xB3, 0x12, 0x0F,
- 0x55, 0x7F, 0x03, 0x12, 0x16, 0xB3, 0x12, 0x00,
-/*1B70*/0x06, 0xE5, 0xE2, 0x30, 0xE7, 0x09, 0x12, 0x10,
- 0x00, 0x30, 0x30, 0x03, 0x12, 0x11, 0x00, 0x02,
-/*1B80*/0x00, 0x47, 0xF0, 0xE5, 0x0B, 0x25, 0xE0, 0x25,
+/*15C0*/0x05, 0x60, 0x02, 0x15, 0x58, 0x22, 0x90, 0x11,
+ 0x4D, 0xE4, 0x93, 0x90, 0x07, 0x2E, 0xF0, 0x12,
+/*15D0*/0x08, 0x1F, 0x75, 0x83, 0xAE, 0xE0, 0x54, 0x1A,
+ 0xF5, 0x34, 0x70, 0x67, 0xEF, 0x44, 0x07, 0xF5,
+/*15E0*/0x82, 0x75, 0x83, 0xCE, 0xE0, 0xFF, 0x13, 0x13,
+ 0x13, 0x54, 0x07, 0xF5, 0x36, 0x54, 0x0F, 0xD3,
+/*15F0*/0x94, 0x00, 0x40, 0x06, 0x12, 0x14, 0x2D, 0x12,
+ 0x1B, 0xA9, 0xE5, 0x36, 0x54, 0x0F, 0x24, 0xFE,
+/*1600*/0x60, 0x0C, 0x14, 0x60, 0x0C, 0x14, 0x60, 0x19,
+ 0x24, 0x03, 0x70, 0x37, 0x80, 0x10, 0x02, 0x1E,
+/*1610*/0x91, 0x12, 0x1E, 0x91, 0x12, 0x07, 0x2A, 0x75,
+ 0x83, 0xCE, 0xE0, 0x54, 0xEF, 0xF0, 0x02, 0x1D,
+/*1620*/0xAE, 0x12, 0x10, 0x14, 0xE4, 0xF5, 0x55, 0x12,
+ 0x1D, 0x85, 0x05, 0x55, 0xE5, 0x55, 0xC3, 0x94,
+/*1630*/0x05, 0x40, 0xF4, 0x12, 0x07, 0x2A, 0x75, 0x83,
+ 0xCE, 0xE0, 0x54, 0xC7, 0x12, 0x07, 0x29, 0xE0,
+/*1640*/0x44, 0x08, 0xF0, 0x22, 0xE4, 0xF5, 0x58, 0xF5,
+ 0x59, 0xAF, 0x08, 0xEF, 0x44, 0x07, 0xF5, 0x82,
+/*1650*/0x75, 0x83, 0xD0, 0xE0, 0xFD, 0xC4, 0x54, 0x0F,
+ 0xF5, 0x5A, 0xEF, 0x44, 0x07, 0xF5, 0x82, 0x75,
+/*1660*/0x83, 0x80, 0x74, 0x01, 0xF0, 0x12, 0x08, 0x21,
+ 0x75, 0x83, 0x82, 0xE5, 0x45, 0xF0, 0xEF, 0x44,
+/*1670*/0x07, 0xF5, 0x82, 0x75, 0x83, 0x8A, 0x74, 0xFF,
+ 0xF0, 0x12, 0x1A, 0x4D, 0x12, 0x07, 0x2A, 0x75,
+/*1680*/0x83, 0xBC, 0xE0, 0x54, 0xEF, 0x12, 0x07, 0x29,
+ 0x75, 0x83, 0xBE, 0xE0, 0x54, 0xEF, 0x12, 0x07,
+/*1690*/0x29, 0x75, 0x83, 0xC0, 0xE0, 0x54, 0xEF, 0x12,
+ 0x07, 0x29, 0x75, 0x83, 0xBC, 0xE0, 0x44, 0x10,
+/*16A0*/0x12, 0x07, 0x29, 0x75, 0x83, 0xBE, 0xE0, 0x44,
+ 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC0, 0xE0,
+/*16B0*/0x44, 0x10, 0xF0, 0xAF, 0x58, 0xE5, 0x59, 0x12,
+ 0x08, 0x78, 0x02, 0x0A, 0xFF, 0xE4, 0xF5, 0x58,
+/*16C0*/0x7D, 0x01, 0xF5, 0x59, 0xAF, 0x35, 0xFE, 0xFC,
+ 0x12, 0x09, 0x15, 0x12, 0x07, 0x2A, 0x75, 0x83,
+/*16D0*/0xB6, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83,
+ 0xB8, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83,
+/*16E0*/0xBA, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83,
+ 0xBC, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83,
+/*16F0*/0xBE, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83,
+ 0xC0, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83,
+/*1700*/0x90, 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2,
+ 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0xE4,
+/*1710*/0x12, 0x07, 0x29, 0x75, 0x83, 0x92, 0xE4, 0x12,
+ 0x07, 0x29, 0x75, 0x83, 0xC6, 0xE4, 0x12, 0x07,
+/*1720*/0x29, 0x75, 0x83, 0xC8, 0xE4, 0xF0, 0xAF, 0x58,
+ 0xFE, 0xE5, 0x59, 0x12, 0x08, 0x7A, 0x02, 0x0A,
+/*1730*/0xFF, 0xE5, 0xE2, 0x30, 0xE4, 0x6C, 0xE5, 0xE7,
+ 0x54, 0xC0, 0x64, 0x40, 0x70, 0x64, 0xE5, 0x09,
+/*1740*/0xC4, 0x54, 0x30, 0xFE, 0xE5, 0x08, 0x25, 0xE0,
+ 0x25, 0xE0, 0x54, 0xC0, 0x4E, 0xFE, 0xEF, 0x54,
+/*1750*/0x3F, 0x4E, 0xFD, 0xE5, 0x2B, 0xAE, 0x2A, 0x78,
+ 0x02, 0xC3, 0x33, 0xCE, 0x33, 0xCE, 0xD8, 0xF9,
+/*1760*/0xF5, 0x82, 0x8E, 0x83, 0xED, 0xF0, 0xE5, 0x2B,
+ 0xAE, 0x2A, 0x78, 0x02, 0xC3, 0x33, 0xCE, 0x33,
+/*1770*/0xCE, 0xD8, 0xF9, 0xFF, 0xF5, 0x82, 0x8E, 0x83,
+ 0xA3, 0xE5, 0xFE, 0xF0, 0x8F, 0x82, 0x8E, 0x83,
+/*1780*/0xA3, 0xA3, 0xE5, 0xFD, 0xF0, 0x8F, 0x82, 0x8E,
+ 0x83, 0xA3, 0xA3, 0xA3, 0xE5, 0xFC, 0xF0, 0xC3,
+/*1790*/0xE5, 0x2B, 0x94, 0xFA, 0xE5, 0x2A, 0x94, 0x00,
+ 0x50, 0x08, 0x05, 0x2B, 0xE5, 0x2B, 0x70, 0x02,
+/*17A0*/0x05, 0x2A, 0x22, 0xE4, 0xFF, 0xE4, 0xF5, 0x58,
+ 0xF5, 0x56, 0xF5, 0x57, 0x74, 0x82, 0xFC, 0x12,
+/*17B0*/0x0E, 0x04, 0x8C, 0x83, 0xE0, 0xF5, 0x10, 0x54,
+ 0x7F, 0xF0, 0xE5, 0x10, 0x44, 0x80, 0x12, 0x0E,
+/*17C0*/0x98, 0xED, 0xF0, 0x7E, 0x0A, 0x12, 0x0E, 0x04,
+ 0x75, 0x83, 0xA0, 0xE0, 0x20, 0xE0, 0x26, 0xDE,
+/*17D0*/0xF4, 0x05, 0x57, 0xE5, 0x57, 0x70, 0x02, 0x05,
+ 0x56, 0xE5, 0x14, 0x24, 0x01, 0xFD, 0xE4, 0x33,
+/*17E0*/0xFC, 0xD3, 0xE5, 0x57, 0x9D, 0xE5, 0x56, 0x9C,
+ 0x40, 0xD9, 0xE5, 0x0A, 0x94, 0x20, 0x50, 0x02,
+/*17F0*/0x05, 0x0A, 0x43, 0xE1, 0x08, 0xC2, 0x31, 0x12,
+ 0x0E, 0x04, 0x75, 0x83, 0xA6, 0xE0, 0x55, 0x12,
+/*1800*/0x65, 0x12, 0x70, 0x03, 0xD2, 0x31, 0x22, 0xC2,
+ 0x31, 0x22, 0x90, 0x07, 0x26, 0xE0, 0xFA, 0xA3,
+/*1810*/0xE0, 0xF5, 0x82, 0x8A, 0x83, 0xE0, 0xF5, 0x41,
+ 0xE5, 0x39, 0xC3, 0x95, 0x41, 0x40, 0x26, 0xE5,
+/*1820*/0x39, 0x95, 0x41, 0xC3, 0x9F, 0xEE, 0x12, 0x07,
+ 0x6B, 0x40, 0x04, 0x7C, 0x01, 0x80, 0x02, 0x7C,
+/*1830*/0x00, 0xE5, 0x41, 0x64, 0x3F, 0x60, 0x04, 0x7B,
+ 0x01, 0x80, 0x02, 0x7B, 0x00, 0xEC, 0x5B, 0x60,
+/*1840*/0x29, 0x05, 0x41, 0x80, 0x28, 0xC3, 0xE5, 0x41,
+ 0x95, 0x39, 0xC3, 0x9F, 0xEE, 0x12, 0x07, 0x6B,
+/*1850*/0x40, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00,
+ 0xE5, 0x41, 0x60, 0x04, 0x7E, 0x01, 0x80, 0x02,
+/*1860*/0x7E, 0x00, 0xEF, 0x5E, 0x60, 0x04, 0x15, 0x41,
+ 0x80, 0x03, 0x85, 0x39, 0x41, 0x85, 0x3A, 0x40,
+/*1870*/0x22, 0xE5, 0xE2, 0x30, 0xE4, 0x60, 0xE5, 0xE1,
+ 0x30, 0xE2, 0x5B, 0xE5, 0x09, 0x70, 0x04, 0x7F,
+/*1880*/0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x08, 0x70,
+ 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE,
+/*1890*/0x5F, 0x60, 0x43, 0x53, 0xF9, 0xF8, 0xE5, 0xE2,
+ 0x30, 0xE4, 0x3B, 0xE5, 0xE1, 0x30, 0xE2, 0x2E,
+/*18A0*/0x43, 0xFA, 0x02, 0x53, 0xFA, 0xFB, 0xE4, 0xF5,
+ 0x10, 0x90, 0x94, 0x70, 0xE5, 0x10, 0xF0, 0xE5,
+/*18B0*/0xE1, 0x30, 0xE2, 0xE7, 0x90, 0x94, 0x70, 0xE0,
+ 0x65, 0x10, 0x60, 0x03, 0x43, 0xFA, 0x04, 0x05,
+/*18C0*/0x10, 0x90, 0x94, 0x70, 0xE5, 0x10, 0xF0, 0x70,
+ 0xE6, 0x12, 0x00, 0x06, 0x80, 0xE1, 0x53, 0xFA,
+/*18D0*/0xFD, 0x53, 0xFA, 0xFB, 0x80, 0xC0, 0x22, 0x8F,
+ 0x54, 0x12, 0x00, 0x06, 0xE5, 0xE1, 0x30, 0xE0,
+/*18E0*/0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5,
+ 0x7E, 0xD3, 0x94, 0x05, 0x40, 0x04, 0x7E, 0x01,
+/*18F0*/0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F, 0x60, 0x3D,
+ 0x85, 0x54, 0x11, 0xE5, 0xE2, 0x20, 0xE1, 0x32,
+/*1900*/0x74, 0xCE, 0x12, 0x1A, 0x05, 0x30, 0xE7, 0x04,
+ 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00, 0x8F, 0x82,
+/*1910*/0x8E, 0x83, 0xE0, 0x30, 0xE6, 0x04, 0x7F, 0x01,
+ 0x80, 0x02, 0x7F, 0x00, 0xEF, 0x5D, 0x70, 0x15,
+/*1920*/0x12, 0x15, 0xC6, 0x74, 0xCE, 0x12, 0x1A, 0x05,
+ 0x30, 0xE6, 0x07, 0xE0, 0x44, 0x80, 0xF0, 0x43,
+/*1930*/0xF9, 0x80, 0x12, 0x18, 0x71, 0x22, 0x12, 0x0E,
+ 0x44, 0xE5, 0x16, 0x25, 0xE0, 0x25, 0xE0, 0x24,
+/*1940*/0xB0, 0xF5, 0x82, 0xE4, 0x34, 0x1A, 0xF5, 0x83,
+ 0xE4, 0x93, 0xF5, 0x0F, 0xE5, 0x16, 0x25, 0xE0,
+/*1950*/0x25, 0xE0, 0x24, 0xB1, 0xF5, 0x82, 0xE4, 0x34,
+ 0x1A, 0xF5, 0x83, 0xE4, 0x93, 0xF5, 0x0E, 0x12,
+/*1960*/0x0E, 0x65, 0xF5, 0x10, 0xE5, 0x0F, 0x54, 0xF0,
+ 0x12, 0x0E, 0x17, 0x75, 0x83, 0x8C, 0xEF, 0xF0,
+/*1970*/0xE5, 0x0F, 0x30, 0xE0, 0x0C, 0x12, 0x0E, 0x04,
+ 0x75, 0x83, 0x86, 0xE0, 0x44, 0x40, 0xF0, 0x80,
+/*1980*/0x0A, 0x12, 0x0E, 0x04, 0x75, 0x83, 0x86, 0xE0,
+ 0x54, 0xBF, 0xF0, 0x12, 0x0E, 0x91, 0x75, 0x83,
+/*1990*/0x82, 0xE5, 0x0E, 0xF0, 0x22, 0x7F, 0x05, 0x12,
+ 0x17, 0x31, 0x12, 0x0E, 0x04, 0x12, 0x0E, 0x33,
+/*19A0*/0x74, 0x02, 0xF0, 0x74, 0x8E, 0xFE, 0x12, 0x0E,
+ 0x04, 0x12, 0x0E, 0x0B, 0xEF, 0xF0, 0x75, 0x15,
+/*19B0*/0x70, 0x12, 0x0F, 0xF7, 0x20, 0x34, 0x05, 0x75,
+ 0x15, 0x10, 0x80, 0x03, 0x75, 0x15, 0x50, 0x12,
+/*19C0*/0x0F, 0xF7, 0x20, 0x34, 0x04, 0x74, 0x10, 0x80,
+ 0x02, 0x74, 0xF0, 0x25, 0x15, 0xF5, 0x15, 0x12,
+/*19D0*/0x0E, 0x21, 0xEF, 0xF0, 0x12, 0x10, 0x91, 0x20,
+ 0x34, 0x17, 0xE5, 0x15, 0x64, 0x30, 0x60, 0x0C,
+/*19E0*/0x74, 0x10, 0x25, 0x15, 0xF5, 0x15, 0xB4, 0x80,
+ 0x03, 0xE4, 0xF5, 0x15, 0x12, 0x0E, 0x21, 0xEF,
+/*19F0*/0xF0, 0x22, 0xF0, 0xE5, 0x0B, 0x25, 0xE0, 0x25,
0xE0, 0x24, 0x82, 0xF5, 0x82, 0xE4, 0x34, 0x07,
-/*1B90*/0xF5, 0x83, 0x22, 0x74, 0x88, 0xFE, 0xE5, 0x08,
+/*1A00*/0xF5, 0x83, 0x22, 0x74, 0x88, 0xFE, 0xE5, 0x08,
0x44, 0x07, 0xFF, 0xF5, 0x82, 0x8E, 0x83, 0xE0,
-/*1BA0*/0x22, 0xF0, 0xE5, 0x08, 0x44, 0x07, 0xF5, 0x82,
+/*1A10*/0x22, 0xF0, 0xE5, 0x08, 0x44, 0x07, 0xF5, 0x82,
0x22, 0xF0, 0xE0, 0x54, 0xC0, 0x8F, 0x82, 0x8E,
-/*1BB0*/0x83, 0xF0, 0x22, 0xEF, 0x44, 0x07, 0xF5, 0x82,
+/*1A20*/0x83, 0xF0, 0x22, 0xEF, 0x44, 0x07, 0xF5, 0x82,
0x75, 0x83, 0x86, 0xE0, 0x54, 0x10, 0xD3, 0x94,
-/*1BC0*/0x00, 0x22, 0xF0, 0x90, 0x07, 0x15, 0xE0, 0x04,
- 0xF0, 0x22, 0xFE, 0xEF, 0x44, 0x07, 0xF5, 0x82,
-/*1BD0*/0x8E, 0x83, 0xE0, 0x22, 0x8E, 0x60, 0x8F, 0x61,
- 0x12, 0x1E, 0x76, 0xE4, 0xFF, 0xCE, 0xED, 0xCE,
-/*1BE0*/0xEE, 0xD3, 0x95, 0x61, 0xE5, 0x60, 0x12, 0x07,
- 0x78, 0x40, 0x39, 0x74, 0x20, 0x2E, 0xF5, 0x82,
-/*1BF0*/0xE4, 0x34, 0x03, 0xF5, 0x83, 0xE0, 0x70, 0x03,
- 0xFF, 0x80, 0x26, 0x12, 0x08, 0xEF, 0xFD, 0xC3,
-/*1C00*/0x9F, 0x40, 0x1E, 0xCF, 0xED, 0xCF, 0xEB, 0x4A,
- 0x70, 0x0B, 0x8D, 0x42, 0x12, 0x08, 0xFB, 0xF5,
-/*1C10*/0x41, 0x8E, 0x40, 0x80, 0x0C, 0x12, 0x08, 0xEF,
- 0xF5, 0x38, 0x12, 0x08, 0xFB, 0xF5, 0x39, 0x8E,
-/*1C20*/0x3A, 0x1E, 0x80, 0xBC, 0x22, 0x75, 0x58, 0x01,
- 0xE5, 0x35, 0x70, 0x0C, 0x12, 0x07, 0xD9, 0xE0,
-/*1C30*/0xF5, 0x4A, 0x12, 0x07, 0xE5, 0xE0, 0xF5, 0x4C,
- 0xE5, 0x35, 0xB4, 0x04, 0x0C, 0x12, 0x07, 0xF1,
-/*1C40*/0xE0, 0xF5, 0x4A, 0x12, 0x07, 0xFD, 0xE0, 0xF5,
- 0x4C, 0xE5, 0x35, 0xB4, 0x01, 0x04, 0x7F, 0x01,
-/*1C50*/0x80, 0x02, 0x7F, 0x00, 0xE5, 0x35, 0xB4, 0x02,
- 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE,
-/*1C60*/0x4F, 0x60, 0x0C, 0x12, 0x08, 0x09, 0xE0, 0xF5,
- 0x4A, 0x12, 0x08, 0x15, 0xE0, 0xF5, 0x4C, 0x85,
-/*1C70*/0x41, 0x49, 0x85, 0x40, 0x4B, 0x22, 0x75, 0x5B,
- 0x01, 0x90, 0x07, 0x24, 0x12, 0x07, 0x42, 0xE0,
-/*1C80*/0x54, 0x1F, 0xFF, 0xD3, 0x94, 0x02, 0x50, 0x04,
- 0x8F, 0x58, 0x80, 0x05, 0xEF, 0x24, 0xFE, 0xF5,
-/*1C90*/0x58, 0xEF, 0xC3, 0x94, 0x18, 0x40, 0x05, 0x75,
- 0x59, 0x18, 0x80, 0x04, 0xEF, 0x04, 0xF5, 0x59,
-/*1CA0*/0x85, 0x43, 0x5A, 0xAF, 0x58, 0x7E, 0x00, 0xAD,
- 0x59, 0x7C, 0x00, 0xAB, 0x5B, 0x7A, 0x00, 0x12,
-/*1CB0*/0x15, 0x41, 0xAF, 0x5A, 0x7E, 0x00, 0x12, 0x17,
- 0x8C, 0xAF, 0x5B, 0x7E, 0x00, 0x02, 0x1A, 0xD8,
-/*1CC0*/0xE4, 0xFC, 0xC3, 0xED, 0x9F, 0xFA, 0xEF, 0xF5,
- 0x83, 0x75, 0x82, 0x00, 0x79, 0xFF, 0xE4, 0x93,
-/*1CD0*/0xCC, 0x6C, 0xCC, 0xA3, 0xD9, 0xF8, 0xDA, 0xF6,
- 0xE5, 0xE2, 0x30, 0xE4, 0x02, 0x8C, 0xE5, 0xED,
-/*1CE0*/0x24, 0xFF, 0xFF, 0xEF, 0x75, 0x82, 0xFF, 0xF5,
- 0x83, 0xE4, 0x93, 0x6C, 0x70, 0x03, 0x7F, 0x01,
-/*1CF0*/0x22, 0x7F, 0x00, 0x22, 0x12, 0x0E, 0xB7, 0x75,
- 0x83, 0x80, 0xE4, 0xF0, 0xE5, 0x08, 0x44, 0x07,
-/*1D00*/0x12, 0x0E, 0xB0, 0x75, 0x83, 0x84, 0x12, 0x0E,
- 0xB5, 0x75, 0x83, 0x86, 0x12, 0x0E, 0xB5, 0x75,
-/*1D10*/0x83, 0x8C, 0xE0, 0x54, 0xF3, 0x12, 0x0E, 0xB6,
- 0x75, 0x83, 0x8E, 0x12, 0x0E, 0xB5, 0x75, 0x83,
-/*1D20*/0x94, 0xE0, 0x54, 0xFB, 0xF0, 0x22, 0x12, 0x07,
- 0x37, 0x75, 0x83, 0x8E, 0xE4, 0x12, 0x07, 0x36,
-/*1D30*/0x74, 0x01, 0x12, 0x07, 0x36, 0xE4, 0x12, 0x08,
- 0xCB, 0x75, 0x83, 0x8C, 0xE0, 0x44, 0x20, 0x12,
-/*1D40*/0x08, 0xCB, 0xE0, 0x54, 0xDF, 0xF0, 0x74, 0x84,
- 0x85, 0x08, 0x82, 0xF5, 0x83, 0xE0, 0x54, 0x7F,
-/*1D50*/0xF0, 0xE0, 0x44, 0x80, 0xF0, 0x22, 0x75, 0x56,
- 0x01, 0xE4, 0xFD, 0xF5, 0x57, 0xAF, 0x35, 0xFE,
-/*1D60*/0xFC, 0x12, 0x09, 0x22, 0x12, 0x1C, 0x76, 0x12,
- 0x1E, 0x4B, 0x12, 0x1C, 0x25, 0xAF, 0x57, 0x7E,
-/*1D70*/0x00, 0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x49,
- 0xAF, 0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE, 0x75,
-/*1D80*/0x56, 0x01, 0xE4, 0xFD, 0xF5, 0x57, 0xAF, 0x35,
- 0xFE, 0xFC, 0x12, 0x09, 0x22, 0x12, 0x1C, 0x76,
-/*1D90*/0x12, 0x1E, 0x4B, 0x12, 0x1C, 0x25, 0xAF, 0x57,
- 0x7E, 0x00, 0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04,
-/*1DA0*/0x49, 0xAF, 0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE,
- 0xE4, 0xF5, 0x16, 0x12, 0x0E, 0xF7, 0xFE, 0xE5,
-/*1DB0*/0x08, 0x44, 0x05, 0xFF, 0x12, 0x0F, 0x18, 0x8F,
- 0x82, 0x8E, 0x83, 0xF0, 0x05, 0x16, 0xE5, 0x16,
-/*1DC0*/0xC3, 0x94, 0x14, 0x40, 0xE6, 0xE5, 0x08, 0x12,
- 0x0E, 0xDE, 0xE4, 0xF0, 0x22, 0xE4, 0xF5, 0x58,
-/*1DD0*/0xF5, 0x59, 0xF5, 0x5A, 0xFF, 0xFE, 0xAD, 0x58,
- 0xFC, 0x12, 0x09, 0x22, 0x7F, 0x04, 0x7E, 0x00,
-/*1DE0*/0xAD, 0x58, 0x7C, 0x00, 0x12, 0x09, 0x22, 0x7F,
- 0x02, 0x7E, 0x00, 0xAD, 0x58, 0x7C, 0x00, 0x02,
-/*1DF0*/0x09, 0x22, 0xE5, 0x3C, 0x25, 0x3E, 0xFC, 0xE5,
- 0x42, 0x24, 0x00, 0xFB, 0xE4, 0x33, 0xFA, 0xEC,
-/*1E00*/0xC3, 0x9B, 0xEA, 0x12, 0x07, 0x78, 0x40, 0x0B,
- 0x8C, 0x42, 0xE5, 0x3D, 0x25, 0x3F, 0xF5, 0x41,
-/*1E10*/0x8F, 0x40, 0x22, 0x12, 0x09, 0x18, 0x22, 0x74,
- 0x84, 0xF5, 0x18, 0x85, 0x08, 0x19, 0x85, 0x19,
-/*1E20*/0x82, 0x85, 0x18, 0x83, 0xE0, 0x54, 0x7F, 0xF0,
- 0xE0, 0x44, 0x80, 0xF0, 0xE0, 0x44, 0x80, 0xF0,
-/*1E30*/0x22, 0xEF, 0x4E, 0x70, 0x0B, 0x12, 0x07, 0x37,
- 0x75, 0x83, 0xD2, 0xE0, 0x54, 0xDF, 0xF0, 0x22,
-/*1E40*/0x12, 0x07, 0x37, 0x75, 0x83, 0xD2, 0xE0, 0x44,
- 0x20, 0xF0, 0x22, 0x75, 0x58, 0x01, 0x90, 0x07,
-/*1E50*/0x26, 0x12, 0x07, 0x42, 0xE0, 0x54, 0x3F, 0xF5,
- 0x41, 0x12, 0x07, 0x3F, 0xE0, 0x54, 0x3F, 0xF5,
-/*1E60*/0x40, 0x22, 0x75, 0x56, 0x02, 0xE4, 0xF5, 0x57,
- 0x12, 0x1D, 0xCD, 0xAF, 0x57, 0x7E, 0x00, 0xAD,
-/*1E70*/0x56, 0x7C, 0x00, 0x02, 0x04, 0x49, 0xE4, 0xF5,
- 0x42, 0xF5, 0x41, 0xF5, 0x40, 0xF5, 0x38, 0xF5,
-/*1E80*/0x39, 0xF5, 0x3A, 0x22, 0xEF, 0x54, 0x07, 0xFF,
- 0xE5, 0xF9, 0x54, 0xF8, 0x4F, 0xF5, 0xF9, 0x22,
-/*1E90*/0x7F, 0x01, 0xE4, 0xFE, 0x0F, 0x0E, 0xBE, 0xFF,
- 0xFB, 0x22, 0x12, 0x0E, 0xD4, 0xEF, 0xF0, 0x12,
-/*1EA0*/0x10, 0x91, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-/*1EB0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-/*1EC0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+/*1A30*/0x00, 0x22, 0xF0, 0x90, 0x07, 0x15, 0xE0, 0x04,
+ 0xF0, 0x22, 0x44, 0x06, 0xF5, 0x82, 0x75, 0x83,
+/*1A40*/0x9E, 0xE0, 0x22, 0xFE, 0xEF, 0x44, 0x07, 0xF5,
+ 0x82, 0x8E, 0x83, 0xE0, 0x22, 0xE4, 0x90, 0x07,
+/*1A50*/0x2A, 0xF0, 0xA3, 0xF0, 0x12, 0x07, 0x2A, 0x75,
+ 0x83, 0x82, 0xE0, 0x54, 0x7F, 0x12, 0x07, 0x29,
+/*1A60*/0xE0, 0x44, 0x80, 0xF0, 0x12, 0x10, 0xFC, 0x12,
+ 0x08, 0x1F, 0x75, 0x83, 0xA0, 0xE0, 0x20, 0xE0,
+/*1A70*/0x1A, 0x90, 0x07, 0x2B, 0xE0, 0x04, 0xF0, 0x70,
+ 0x06, 0x90, 0x07, 0x2A, 0xE0, 0x04, 0xF0, 0x90,
+/*1A80*/0x07, 0x2A, 0xE0, 0xB4, 0x10, 0xE1, 0xA3, 0xE0,
+ 0xB4, 0x00, 0xDC, 0xEE, 0x44, 0xA6, 0xFC, 0xEF,
+/*1A90*/0x44, 0x07, 0xF5, 0x82, 0x8C, 0x83, 0xE0, 0xF5,
+ 0x32, 0xEE, 0x44, 0xA8, 0xFE, 0xEF, 0x44, 0x07,
+/*1AA0*/0xF5, 0x82, 0x8E, 0x83, 0xE0, 0xF5, 0x33, 0x22,
+ 0x01, 0x20, 0x11, 0x00, 0x04, 0x20, 0x00, 0x90,
+/*1AB0*/0x00, 0x20, 0x0F, 0x92, 0x00, 0x21, 0x0F, 0x94,
+ 0x00, 0x22, 0x0F, 0x96, 0x00, 0x23, 0x0F, 0x98,
+/*1AC0*/0x00, 0x24, 0x0F, 0x9A, 0x00, 0x25, 0x0F, 0x9C,
+ 0x00, 0x26, 0x0F, 0x9E, 0x00, 0x27, 0x0F, 0xA0,
+/*1AD0*/0x01, 0x20, 0x01, 0xA2, 0x01, 0x21, 0x01, 0xA4,
+ 0x01, 0x22, 0x01, 0xA6, 0x01, 0x23, 0x01, 0xA8,
+/*1AE0*/0x01, 0x24, 0x01, 0xAA, 0x01, 0x25, 0x01, 0xAC,
+ 0x01, 0x26, 0x01, 0xAE, 0x01, 0x27, 0x01, 0xB0,
+/*1AF0*/0x01, 0x28, 0x01, 0xB4, 0x00, 0x28, 0x0F, 0xB6,
+ 0x40, 0x28, 0x0F, 0xB8, 0x61, 0x28, 0x01, 0xCB,
+/*1B00*/0xEF, 0xCB, 0xCA, 0xEE, 0xCA, 0x7F, 0x01, 0xE4,
+ 0xFD, 0xEB, 0x4A, 0x70, 0x24, 0xE5, 0x08, 0xF5,
+/*1B10*/0x82, 0x74, 0xB6, 0x12, 0x08, 0x29, 0xE5, 0x08,
+ 0xF5, 0x82, 0x74, 0xB8, 0x12, 0x08, 0x29, 0xE5,
+/*1B20*/0x08, 0xF5, 0x82, 0x74, 0xBA, 0x12, 0x08, 0x29,
+ 0x7E, 0x00, 0x7C, 0x00, 0x12, 0x0A, 0xFF, 0x80,
+/*1B30*/0x12, 0x90, 0x07, 0x26, 0x12, 0x07, 0x35, 0xE5,
+ 0x41, 0xF0, 0x90, 0x07, 0x24, 0x12, 0x07, 0x35,
+/*1B40*/0xE5, 0x40, 0xF0, 0x12, 0x07, 0x2A, 0x75, 0x83,
+ 0x8E, 0xE4, 0x12, 0x07, 0x29, 0x74, 0x01, 0x12,
+/*1B50*/0x07, 0x29, 0xE4, 0xF0, 0x22, 0xE4, 0xF5, 0x26,
+ 0xF5, 0x27, 0x53, 0xE1, 0xFE, 0xF5, 0x2A, 0x75,
+/*1B60*/0x2B, 0x01, 0xF5, 0x08, 0x7F, 0x01, 0x12, 0x17,
+ 0x31, 0x30, 0x30, 0x1C, 0x90, 0x1A, 0xA9, 0xE4,
+/*1B70*/0x93, 0xF5, 0x10, 0x90, 0x1F, 0xF9, 0xE4, 0x93,
+ 0xF5, 0x10, 0x90, 0x00, 0x41, 0xE4, 0x93, 0xF5,
+/*1B80*/0x10, 0x90, 0x1E, 0xCA, 0xE4, 0x93, 0xF5, 0x10,
+ 0x7F, 0x02, 0x12, 0x17, 0x31, 0x12, 0x0F, 0x54,
+/*1B90*/0x7F, 0x03, 0x12, 0x17, 0x31, 0x12, 0x00, 0x06,
+ 0xE5, 0xE2, 0x30, 0xE7, 0x09, 0x12, 0x10, 0x00,
+/*1BA0*/0x30, 0x30, 0x03, 0x12, 0x11, 0x00, 0x02, 0x00,
+ 0x47, 0x12, 0x08, 0x1F, 0x75, 0x83, 0xD0, 0xE0,
+/*1BB0*/0xC4, 0x54, 0x0F, 0xFD, 0x75, 0x43, 0x01, 0x75,
+ 0x44, 0xFF, 0x12, 0x08, 0xAA, 0x74, 0x04, 0xF0,
+/*1BC0*/0x75, 0x3B, 0x01, 0xED, 0x14, 0x60, 0x0C, 0x14,
+ 0x60, 0x0B, 0x14, 0x60, 0x0F, 0x24, 0x03, 0x70,
+/*1BD0*/0x0B, 0x80, 0x09, 0x80, 0x00, 0x12, 0x08, 0xA7,
+ 0x04, 0xF0, 0x80, 0x06, 0x12, 0x08, 0xA7, 0x74,
+/*1BE0*/0x04, 0xF0, 0xEE, 0x44, 0x82, 0xFE, 0xEF, 0x44,
+ 0x07, 0xF5, 0x82, 0x8E, 0x83, 0xE5, 0x45, 0x12,
+/*1BF0*/0x08, 0xBE, 0x75, 0x83, 0x82, 0xE5, 0x31, 0xF0,
+ 0x02, 0x11, 0x4C, 0x8E, 0x60, 0x8F, 0x61, 0x12,
+/*1C00*/0x1E, 0xA5, 0xE4, 0xFF, 0xCE, 0xED, 0xCE, 0xEE,
+ 0xD3, 0x95, 0x61, 0xE5, 0x60, 0x12, 0x07, 0x6B,
+/*1C10*/0x40, 0x39, 0x74, 0x20, 0x2E, 0xF5, 0x82, 0xE4,
+ 0x34, 0x03, 0xF5, 0x83, 0xE0, 0x70, 0x03, 0xFF,
+/*1C20*/0x80, 0x26, 0x12, 0x08, 0xE2, 0xFD, 0xC3, 0x9F,
+ 0x40, 0x1E, 0xCF, 0xED, 0xCF, 0xEB, 0x4A, 0x70,
+/*1C30*/0x0B, 0x8D, 0x42, 0x12, 0x08, 0xEE, 0xF5, 0x41,
+ 0x8E, 0x40, 0x80, 0x0C, 0x12, 0x08, 0xE2, 0xF5,
+/*1C40*/0x38, 0x12, 0x08, 0xEE, 0xF5, 0x39, 0x8E, 0x3A,
+ 0x1E, 0x80, 0xBC, 0x22, 0x75, 0x58, 0x01, 0xE5,
+/*1C50*/0x35, 0x70, 0x0C, 0x12, 0x07, 0xCC, 0xE0, 0xF5,
+ 0x4A, 0x12, 0x07, 0xD8, 0xE0, 0xF5, 0x4C, 0xE5,
+/*1C60*/0x35, 0xB4, 0x04, 0x0C, 0x12, 0x07, 0xE4, 0xE0,
+ 0xF5, 0x4A, 0x12, 0x07, 0xF0, 0xE0, 0xF5, 0x4C,
+/*1C70*/0xE5, 0x35, 0xB4, 0x01, 0x04, 0x7F, 0x01, 0x80,
+ 0x02, 0x7F, 0x00, 0xE5, 0x35, 0xB4, 0x02, 0x04,
+/*1C80*/0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F,
+ 0x60, 0x0C, 0x12, 0x07, 0xFC, 0xE0, 0xF5, 0x4A,
+/*1C90*/0x12, 0x08, 0x08, 0xE0, 0xF5, 0x4C, 0x85, 0x41,
+ 0x49, 0x85, 0x40, 0x4B, 0x22, 0x75, 0x5B, 0x01,
+/*1CA0*/0x90, 0x07, 0x24, 0x12, 0x07, 0x35, 0xE0, 0x54,
+ 0x1F, 0xFF, 0xD3, 0x94, 0x02, 0x50, 0x04, 0x8F,
+/*1CB0*/0x58, 0x80, 0x05, 0xEF, 0x24, 0xFE, 0xF5, 0x58,
+ 0xEF, 0xC3, 0x94, 0x18, 0x40, 0x05, 0x75, 0x59,
+/*1CC0*/0x18, 0x80, 0x04, 0xEF, 0x04, 0xF5, 0x59, 0x85,
+ 0x43, 0x5A, 0xAF, 0x58, 0x7E, 0x00, 0xAD, 0x59,
+/*1CD0*/0x7C, 0x00, 0xAB, 0x5B, 0x7A, 0x00, 0x12, 0x15,
+ 0x41, 0xAF, 0x5A, 0x7E, 0x00, 0x12, 0x18, 0x0A,
+/*1CE0*/0xAF, 0x5B, 0x7E, 0x00, 0x02, 0x1A, 0xFF, 0xE5,
+ 0xE2, 0x30, 0xE7, 0x0E, 0x12, 0x10, 0x03, 0xC2,
+/*1CF0*/0x30, 0x30, 0x30, 0x03, 0x12, 0x10, 0xFF, 0x20,
+ 0x33, 0x28, 0xE5, 0xE7, 0x30, 0xE7, 0x05, 0x12,
+/*1D00*/0x0E, 0xA2, 0x80, 0x0D, 0xE5, 0xFE, 0xC3, 0x94,
+ 0x20, 0x50, 0x06, 0x12, 0x0E, 0xA2, 0x43, 0xF9,
+/*1D10*/0x08, 0xE5, 0xF2, 0x30, 0xE7, 0x03, 0x53, 0xF9,
+ 0x7F, 0xE5, 0xF1, 0x54, 0x70, 0xD3, 0x94, 0x00,
+/*1D20*/0x50, 0xD8, 0x22, 0x12, 0x0E, 0x04, 0x75, 0x83,
+ 0x80, 0xE4, 0xF0, 0xE5, 0x08, 0x44, 0x07, 0x12,
+/*1D30*/0x0D, 0xFD, 0x75, 0x83, 0x84, 0x12, 0x0E, 0x02,
+ 0x75, 0x83, 0x86, 0x12, 0x0E, 0x02, 0x75, 0x83,
+/*1D40*/0x8C, 0xE0, 0x54, 0xF3, 0x12, 0x0E, 0x03, 0x75,
+ 0x83, 0x8E, 0x12, 0x0E, 0x02, 0x75, 0x83, 0x94,
+/*1D50*/0xE0, 0x54, 0xFB, 0xF0, 0x22, 0x12, 0x07, 0x2A,
+ 0x75, 0x83, 0x8E, 0xE4, 0x12, 0x07, 0x29, 0x74,
+/*1D60*/0x01, 0x12, 0x07, 0x29, 0xE4, 0x12, 0x08, 0xBE,
+ 0x75, 0x83, 0x8C, 0xE0, 0x44, 0x20, 0x12, 0x08,
+/*1D70*/0xBE, 0xE0, 0x54, 0xDF, 0xF0, 0x74, 0x84, 0x85,
+ 0x08, 0x82, 0xF5, 0x83, 0xE0, 0x54, 0x7F, 0xF0,
+/*1D80*/0xE0, 0x44, 0x80, 0xF0, 0x22, 0x75, 0x56, 0x01,
+ 0xE4, 0xFD, 0xF5, 0x57, 0xAF, 0x35, 0xFE, 0xFC,
+/*1D90*/0x12, 0x09, 0x15, 0x12, 0x1C, 0x9D, 0x12, 0x1E,
+ 0x7A, 0x12, 0x1C, 0x4C, 0xAF, 0x57, 0x7E, 0x00,
+/*1DA0*/0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x44, 0xAF,
+ 0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE, 0x75, 0x56,
+/*1DB0*/0x01, 0xE4, 0xFD, 0xF5, 0x57, 0xAF, 0x35, 0xFE,
+ 0xFC, 0x12, 0x09, 0x15, 0x12, 0x1C, 0x9D, 0x12,
+/*1DC0*/0x1E, 0x7A, 0x12, 0x1C, 0x4C, 0xAF, 0x57, 0x7E,
+ 0x00, 0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x44,
+/*1DD0*/0xAF, 0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE, 0xE4,
+ 0xF5, 0x16, 0x12, 0x0E, 0x44, 0xFE, 0xE5, 0x08,
+/*1DE0*/0x44, 0x05, 0xFF, 0x12, 0x0E, 0x65, 0x8F, 0x82,
+ 0x8E, 0x83, 0xF0, 0x05, 0x16, 0xE5, 0x16, 0xC3,
+/*1DF0*/0x94, 0x14, 0x40, 0xE6, 0xE5, 0x08, 0x12, 0x0E,
+ 0x2B, 0xE4, 0xF0, 0x22, 0xE4, 0xF5, 0x58, 0xF5,
+/*1E00*/0x59, 0xF5, 0x5A, 0xFF, 0xFE, 0xAD, 0x58, 0xFC,
+ 0x12, 0x09, 0x15, 0x7F, 0x04, 0x7E, 0x00, 0xAD,
+/*1E10*/0x58, 0x7C, 0x00, 0x12, 0x09, 0x15, 0x7F, 0x02,
+ 0x7E, 0x00, 0xAD, 0x58, 0x7C, 0x00, 0x02, 0x09,
+/*1E20*/0x15, 0xE5, 0x3C, 0x25, 0x3E, 0xFC, 0xE5, 0x42,
+ 0x24, 0x00, 0xFB, 0xE4, 0x33, 0xFA, 0xEC, 0xC3,
+/*1E30*/0x9B, 0xEA, 0x12, 0x07, 0x6B, 0x40, 0x0B, 0x8C,
+ 0x42, 0xE5, 0x3D, 0x25, 0x3F, 0xF5, 0x41, 0x8F,
+/*1E40*/0x40, 0x22, 0x12, 0x09, 0x0B, 0x22, 0x74, 0x84,
+ 0xF5, 0x18, 0x85, 0x08, 0x19, 0x85, 0x19, 0x82,
+/*1E50*/0x85, 0x18, 0x83, 0xE0, 0x54, 0x7F, 0xF0, 0xE0,
+ 0x44, 0x80, 0xF0, 0xE0, 0x44, 0x80, 0xF0, 0x22,
+/*1E60*/0xEF, 0x4E, 0x70, 0x0B, 0x12, 0x07, 0x2A, 0x75,
+ 0x83, 0xD2, 0xE0, 0x54, 0xDF, 0xF0, 0x22, 0x12,
+/*1E70*/0x07, 0x2A, 0x75, 0x83, 0xD2, 0xE0, 0x44, 0x20,
+ 0xF0, 0x22, 0x75, 0x58, 0x01, 0x90, 0x07, 0x26,
+/*1E80*/0x12, 0x07, 0x35, 0xE0, 0x54, 0x3F, 0xF5, 0x41,
+ 0x12, 0x07, 0x32, 0xE0, 0x54, 0x3F, 0xF5, 0x40,
+/*1E90*/0x22, 0x75, 0x56, 0x02, 0xE4, 0xF5, 0x57, 0x12,
+ 0x1D, 0xFC, 0xAF, 0x57, 0x7E, 0x00, 0xAD, 0x56,
+/*1EA0*/0x7C, 0x00, 0x02, 0x04, 0x44, 0xE4, 0xF5, 0x42,
+ 0xF5, 0x41, 0xF5, 0x40, 0xF5, 0x38, 0xF5, 0x39,
+/*1EB0*/0xF5, 0x3A, 0x22, 0xEF, 0x54, 0x07, 0xFF, 0xE5,
+ 0xF9, 0x54, 0xF8, 0x4F, 0xF5, 0xF9, 0x22, 0x7F,
+/*1EC0*/0x01, 0xE4, 0xFE, 0x0F, 0x0E, 0xBE, 0xFF, 0xFB,
+ 0x22, 0x01, 0x20, 0x00, 0x01, 0x04, 0x20, 0x00,
/*1ED0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
/*1EE0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -1066,7 +1066,7 @@ static unsigned char ipath_sd7220_ib_img[] = {
/*1FE0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
/*1FF0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x01, 0x1F, 0x10, 0x00, 0x04, 0x20, 0x00, 0xC0
+ 0x01, 0x20, 0x11, 0x00, 0x04, 0x20, 0x00, 0x81
};
int ipath_sd7220_ib_load(struct ipath_devdata *dd)
diff --git a/drivers/infiniband/hw/ipath/ipath_sdma.c b/drivers/infiniband/hw/ipath/ipath_sdma.c
index 33a1c7c..3697449 100644
--- a/drivers/infiniband/hw/ipath/ipath_sdma.c
+++ b/drivers/infiniband/hw/ipath/ipath_sdma.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -230,7 +230,6 @@ static void dump_sdma_state(struct ipath_devdata *dd)
static void sdma_abort_task(unsigned long opaque)
{
struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
- int kick = 0;
u64 status;
unsigned long flags;
@@ -247,7 +246,7 @@ static void sdma_abort_task(unsigned long opaque)
/* ipath_sdma_abort() is done, waiting for interrupt */
if (status == IPATH_SDMA_ABORT_DISARMED) {
- if (dd->ipath_sdma_abort_intr_timeout < jiffies)
+ if (jiffies < dd->ipath_sdma_abort_intr_timeout)
goto resched_noprint;
/* give up, intr got lost somewhere */
ipath_dbg("give up waiting for SDMADISABLED intr\n");
@@ -308,30 +307,35 @@ static void sdma_abort_task(unsigned long opaque)
/* done with sdma state for a bit */
spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
- /* restart sdma engine */
+ /*
+ * Don't restart sdma here (with the exception
+ * below). Wait until link is up to ACTIVE. VL15 MADs
+ * used to bring the link up use PIO, and multiple link
+ * transitions otherwise cause the sdma engine to be
+ * stopped and started multiple times.
+ * The disable is done here, including the shadow,
+ * so the state is kept consistent.
+ * See ipath_restart_sdma() for the actual starting
+ * of sdma.
+ */
spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
dd->ipath_sendctrl);
ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
- dd->ipath_sendctrl |= INFINIPATH_S_SDMAENABLE;
- ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
- dd->ipath_sendctrl);
- ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
- kick = 1;
- ipath_dbg("sdma restarted from abort\n");
-
- /* now clear status bits */
- spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
- __clear_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
- __clear_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
- __clear_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
/* make sure I see next message */
dd->ipath_sdma_abort_jiffies = 0;
- goto unlock;
+ /*
+ * Not everything that takes SDMA offline is a link
+ * status change. If the link was up, restart SDMA.
+ */
+ if (dd->ipath_flags & IPATH_LINKACTIVE)
+ ipath_restart_sdma(dd);
+
+ goto done;
}
resched:
@@ -353,10 +357,8 @@ resched_noprint:
unlock:
spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
-
- /* kick upper layers */
- if (kick)
- ipath_ib_piobufavail(dd->verbs_dev);
+done:
+ return;
}
/*
@@ -434,7 +436,12 @@ int setup_sdma(struct ipath_devdata *dd)
goto done;
}
- dd->ipath_sdma_status = 0;
+ /*
+ * Set initial status as if we had been up, then gone down.
+ * This lets initial start on transition to ACTIVE be the
+ * same as restart after link flap.
+ */
+ dd->ipath_sdma_status = IPATH_SDMA_ABORT_ABORTED;
dd->ipath_sdma_abort_jiffies = 0;
dd->ipath_sdma_generation = 0;
dd->ipath_sdma_descq_tail = 0;
@@ -456,16 +463,19 @@ int setup_sdma(struct ipath_devdata *dd)
ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr,
dd->ipath_sdma_head_phys);
- /* Reserve all the former "kernel" piobufs */
- n = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k - dd->ipath_pioreserved;
- for (i = dd->ipath_lastport_piobuf; i < n; ++i) {
+ /*
+ * Reserve all the former "kernel" piobufs, using high number range
+ * so we get as many 4K buffers as possible
+ */
+ n = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k;
+ i = dd->ipath_lastport_piobuf + dd->ipath_pioreserved;
+ ipath_chg_pioavailkernel(dd, i, n - i , 0);
+ for (; i < n; ++i) {
unsigned word = i / 64;
unsigned bit = i & 63;
BUG_ON(word >= 3);
senddmabufmask[word] |= 1ULL << bit;
}
- ipath_chg_pioavailkernel(dd, dd->ipath_lastport_piobuf,
- n - dd->ipath_lastport_piobuf, 0);
ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0,
senddmabufmask[0]);
ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1,
@@ -481,10 +491,14 @@ int setup_sdma(struct ipath_devdata *dd)
tasklet_init(&dd->ipath_sdma_abort_task, sdma_abort_task,
(unsigned long) dd);
- /* Turn on SDMA */
+ /*
+ * No use to turn on SDMA here, as link is probably not ACTIVE
+ * Just mark it RUNNING and enable the interrupt, and let the
+ * ipath_restart_sdma() on link transition to ACTIVE actually
+ * enable it.
+ */
spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
- dd->ipath_sendctrl |= INFINIPATH_S_SDMAENABLE |
- INFINIPATH_S_SDMAINTENABLE;
+ dd->ipath_sendctrl |= INFINIPATH_S_SDMAINTENABLE;
ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
__set_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status);
@@ -572,6 +586,59 @@ void teardown_sdma(struct ipath_devdata *dd)
sdma_descq, sdma_descq_phys);
}
+/*
+ * [Re]start SDMA, if we use it, and it's not already OK.
+ * This is called on transition to link ACTIVE, either the first or
+ * subsequent times.
+ */
+void ipath_restart_sdma(struct ipath_devdata *dd)
+{
+ unsigned long flags;
+ int needed = 1;
+
+ if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA))
+ goto bail;
+
+ /*
+ * First, make sure we should, which is to say,
+ * check that we are "RUNNING" (not in teardown)
+ * and not "SHUTDOWN"
+ */
+ spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
+ if (!test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status)
+ || test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
+ needed = 0;
+ else {
+ __clear_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
+ __clear_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
+ __clear_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
+ }
+ spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
+ if (!needed) {
+ ipath_dbg("invalid attempt to restart SDMA, status 0x%016llx\n",
+ dd->ipath_sdma_status);
+ goto bail;
+ }
+ spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
+ /*
+ * First clear, just to be safe. Enable is only done
+ * in chip on 0->1 transition
+ */
+ dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
+ ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
+ dd->ipath_sendctrl |= INFINIPATH_S_SDMAENABLE;
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
+ ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
+ spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
+
+ /* notify upper layers */
+ ipath_ib_piobufavail(dd->verbs_dev);
+
+bail:
+ return;
+}
+
static inline void make_sdma_desc(struct ipath_devdata *dd,
u64 *sdmadesc, u64 addr, u64 dwlen, u64 dwoffset)
{
diff --git a/drivers/infiniband/hw/ipath/ipath_srq.c b/drivers/infiniband/hw/ipath/ipath_srq.c
index cc006ba..d9608d0 100644
--- a/drivers/infiniband/hw/ipath/ipath_srq.c
+++ b/drivers/infiniband/hw/ipath/ipath_srq.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
diff --git a/drivers/infiniband/hw/ipath/ipath_stats.c b/drivers/infiniband/hw/ipath/ipath_stats.c
index 643ec29..b6c809d 100644
--- a/drivers/infiniband/hw/ipath/ipath_stats.c
+++ b/drivers/infiniband/hw/ipath/ipath_stats.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -292,7 +292,7 @@ void ipath_get_faststats(unsigned long opaque)
&& time_after(jiffies, dd->ipath_unmasktime)) {
char ebuf[256];
int iserr;
- iserr = ipath_decode_err(ebuf, sizeof ebuf,
+ iserr = ipath_decode_err(dd, ebuf, sizeof ebuf,
dd->ipath_maskederrs);
if (dd->ipath_maskederrs &
~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL |
diff --git a/drivers/infiniband/hw/ipath/ipath_sysfs.c b/drivers/infiniband/hw/ipath/ipath_sysfs.c
index c4266d0..ca1df73 100644
--- a/drivers/infiniband/hw/ipath/ipath_sysfs.c
+++ b/drivers/infiniband/hw/ipath/ipath_sysfs.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
diff --git a/drivers/infiniband/hw/ipath/ipath_uc.c b/drivers/infiniband/hw/ipath/ipath_uc.c
index 2dd8de2..717c13e 100644
--- a/drivers/infiniband/hw/ipath/ipath_uc.c
+++ b/drivers/infiniband/hw/ipath/ipath_uc.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -47,14 +47,30 @@ int ipath_make_uc_req(struct ipath_qp *qp)
{
struct ipath_other_headers *ohdr;
struct ipath_swqe *wqe;
+ unsigned long flags;
u32 hwords;
u32 bth0;
u32 len;
u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
int ret = 0;
- if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK))
+ spin_lock_irqsave(&qp->s_lock, flags);
+
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) {
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))
+ goto bail;
+ /* We are in the error state, flush the work request. */
+ if (qp->s_last == qp->s_head)
+ goto bail;
+ /* If DMAs are in progress, we can't flush immediately. */
+ if (atomic_read(&qp->s_dma_busy)) {
+ qp->s_flags |= IPATH_S_WAIT_DMA;
+ goto bail;
+ }
+ wqe = get_swqe_ptr(qp, qp->s_last);
+ ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
goto done;
+ }
ohdr = &qp->s_hdr.u.oth;
if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
@@ -69,9 +85,12 @@ int ipath_make_uc_req(struct ipath_qp *qp)
qp->s_wqe = NULL;
switch (qp->s_state) {
default:
+ if (!(ib_ipath_state_ops[qp->state] &
+ IPATH_PROCESS_NEXT_SEND_OK))
+ goto bail;
/* Check if send work queue is empty. */
if (qp->s_cur == qp->s_head)
- goto done;
+ goto bail;
/*
* Start a new request.
*/
@@ -79,6 +98,7 @@ int ipath_make_uc_req(struct ipath_qp *qp)
qp->s_sge.sge = wqe->sg_list[0];
qp->s_sge.sg_list = wqe->sg_list + 1;
qp->s_sge.num_sge = wqe->wr.num_sge;
+ qp->s_sge.total_len = wqe->length;
qp->s_len = len = wqe->length;
switch (wqe->wr.opcode) {
case IB_WR_SEND:
@@ -134,7 +154,7 @@ int ipath_make_uc_req(struct ipath_qp *qp)
break;
default:
- goto done;
+ goto bail;
}
break;
@@ -194,9 +214,14 @@ int ipath_make_uc_req(struct ipath_qp *qp)
ipath_make_ruc_header(to_idev(qp->ibqp.device),
qp, ohdr, bth0 | (qp->s_state << 24),
qp->s_next_psn++ & IPATH_PSN_MASK);
+done:
ret = 1;
+ goto unlock;
-done:
+bail:
+ qp->s_flags &= ~IPATH_S_BUSY;
+unlock:
+ spin_unlock_irqrestore(&qp->s_lock, flags);
return ret;
}
@@ -258,8 +283,7 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
*/
opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
- wc.imm_data = 0;
- wc.wc_flags = 0;
+ memset(&wc, 0, sizeof wc);
/* Compare the PSN verses the expected PSN. */
if (unlikely(ipath_cmp24(psn, qp->r_psn) != 0)) {
@@ -322,8 +346,8 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
case OP(SEND_ONLY):
case OP(SEND_ONLY_WITH_IMMEDIATE):
send_first:
- if (qp->r_reuse_sge) {
- qp->r_reuse_sge = 0;
+ if (qp->r_flags & IPATH_R_REUSE_SGE) {
+ qp->r_flags &= ~IPATH_R_REUSE_SGE;
qp->r_sge = qp->s_rdma_read_sge;
} else if (!ipath_get_rwqe(qp, 0)) {
dev->n_pkt_drops++;
@@ -340,13 +364,13 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
case OP(SEND_MIDDLE):
/* Check for invalid length PMTU or posted rwqe len. */
if (unlikely(tlen != (hdrsize + pmtu + 4))) {
- qp->r_reuse_sge = 1;
+ qp->r_flags |= IPATH_R_REUSE_SGE;
dev->n_pkt_drops++;
goto done;
}
qp->r_rcv_len += pmtu;
if (unlikely(qp->r_rcv_len > qp->r_len)) {
- qp->r_reuse_sge = 1;
+ qp->r_flags |= IPATH_R_REUSE_SGE;
dev->n_pkt_drops++;
goto done;
}
@@ -372,7 +396,7 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
/* Check for invalid length. */
/* XXX LAST len should be >= 1 */
if (unlikely(tlen < (hdrsize + pad + 4))) {
- qp->r_reuse_sge = 1;
+ qp->r_flags |= IPATH_R_REUSE_SGE;
dev->n_pkt_drops++;
goto done;
}
@@ -380,24 +404,19 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
tlen -= (hdrsize + pad + 4);
wc.byte_len = tlen + qp->r_rcv_len;
if (unlikely(wc.byte_len > qp->r_len)) {
- qp->r_reuse_sge = 1;
+ qp->r_flags |= IPATH_R_REUSE_SGE;
dev->n_pkt_drops++;
goto done;
}
- /* XXX Need to free SGEs */
+ wc.opcode = IB_WC_RECV;
last_imm:
ipath_copy_sge(&qp->r_sge, data, tlen);
wc.wr_id = qp->r_wr_id;
wc.status = IB_WC_SUCCESS;
- wc.opcode = IB_WC_RECV;
- wc.vendor_err = 0;
wc.qp = &qp->ibqp;
wc.src_qp = qp->remote_qpn;
- wc.pkey_index = 0;
wc.slid = qp->remote_ah_attr.dlid;
wc.sl = qp->remote_ah_attr.sl;
- wc.dlid_path_bits = 0;
- wc.port_num = 0;
/* Signal completion event if the solicited bit is set. */
ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
(ohdr->bth[0] &
@@ -488,13 +507,14 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
dev->n_pkt_drops++;
goto done;
}
- if (qp->r_reuse_sge)
- qp->r_reuse_sge = 0;
+ if (qp->r_flags & IPATH_R_REUSE_SGE)
+ qp->r_flags &= ~IPATH_R_REUSE_SGE;
else if (!ipath_get_rwqe(qp, 1)) {
dev->n_pkt_drops++;
goto done;
}
wc.byte_len = qp->r_len;
+ wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
goto last_imm;
case OP(RDMA_WRITE_LAST):
diff --git a/drivers/infiniband/hw/ipath/ipath_ud.c b/drivers/infiniband/hw/ipath/ipath_ud.c
index 4d4d58d..64e0c9a 100644
--- a/drivers/infiniband/hw/ipath/ipath_ud.c
+++ b/drivers/infiniband/hw/ipath/ipath_ud.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -65,9 +65,9 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe)
u32 length;
qp = ipath_lookup_qpn(&dev->qp_table, swqe->wr.wr.ud.remote_qpn);
- if (!qp) {
+ if (!qp || !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
dev->n_pkt_drops++;
- goto send_comp;
+ goto done;
}
rsge.sg_list = NULL;
@@ -91,14 +91,12 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe)
* present on the wire.
*/
length = swqe->length;
+ memset(&wc, 0, sizeof wc);
wc.byte_len = length + sizeof(struct ib_grh);
if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
wc.wc_flags = IB_WC_WITH_IMM;
wc.imm_data = swqe->wr.imm_data;
- } else {
- wc.wc_flags = 0;
- wc.imm_data = 0;
}
/*
@@ -229,7 +227,6 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe)
}
wc.status = IB_WC_SUCCESS;
wc.opcode = IB_WC_RECV;
- wc.vendor_err = 0;
wc.qp = &qp->ibqp;
wc.src_qp = sqp->ibqp.qp_num;
/* XXX do we know which pkey matched? Only needed for GSI. */
@@ -248,8 +245,7 @@ drop:
kfree(rsge.sg_list);
if (atomic_dec_and_test(&qp->refcount))
wake_up(&qp->wait);
-send_comp:
- ipath_send_complete(sqp, swqe, IB_WC_SUCCESS);
+done:;
}
/**
@@ -264,6 +260,7 @@ int ipath_make_ud_req(struct ipath_qp *qp)
struct ipath_other_headers *ohdr;
struct ib_ah_attr *ah_attr;
struct ipath_swqe *wqe;
+ unsigned long flags;
u32 nwords;
u32 extra_bytes;
u32 bth0;
@@ -271,13 +268,30 @@ int ipath_make_ud_req(struct ipath_qp *qp)
u16 lid;
int ret = 0;
- if (unlikely(!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)))
- goto bail;
+ spin_lock_irqsave(&qp->s_lock, flags);
+
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_NEXT_SEND_OK)) {
+ if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))
+ goto bail;
+ /* We are in the error state, flush the work request. */
+ if (qp->s_last == qp->s_head)
+ goto bail;
+ /* If DMAs are in progress, we can't flush immediately. */
+ if (atomic_read(&qp->s_dma_busy)) {
+ qp->s_flags |= IPATH_S_WAIT_DMA;
+ goto bail;
+ }
+ wqe = get_swqe_ptr(qp, qp->s_last);
+ ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
+ goto done;
+ }
if (qp->s_cur == qp->s_head)
goto bail;
wqe = get_swqe_ptr(qp, qp->s_cur);
+ if (++qp->s_cur >= qp->s_size)
+ qp->s_cur = 0;
/* Construct the header. */
ah_attr = &to_iah(wqe->wr.wr.ud.ah)->attr;
@@ -288,10 +302,23 @@ int ipath_make_ud_req(struct ipath_qp *qp)
dev->n_unicast_xmit++;
} else {
dev->n_unicast_xmit++;
- lid = ah_attr->dlid &
- ~((1 << dev->dd->ipath_lmc) - 1);
+ lid = ah_attr->dlid & ~((1 << dev->dd->ipath_lmc) - 1);
if (unlikely(lid == dev->dd->ipath_lid)) {
+ /*
+ * If DMAs are in progress, we can't generate
+ * a completion for the loopback packet since
+ * it would be out of order.
+ * XXX Instead of waiting, we could queue a
+ * zero length descriptor so we get a callback.
+ */
+ if (atomic_read(&qp->s_dma_busy)) {
+ qp->s_flags |= IPATH_S_WAIT_DMA;
+ goto bail;
+ }
+ spin_unlock_irqrestore(&qp->s_lock, flags);
ipath_ud_loopback(qp, wqe);
+ spin_lock_irqsave(&qp->s_lock, flags);
+ ipath_send_complete(qp, wqe, IB_WC_SUCCESS);
goto done;
}
}
@@ -308,6 +335,7 @@ int ipath_make_ud_req(struct ipath_qp *qp)
qp->s_sge.sge = wqe->sg_list[0];
qp->s_sge.sg_list = wqe->sg_list + 1;
qp->s_sge.num_sge = wqe->wr.num_sge;
+ qp->s_sge.total_len = wqe->length;
if (ah_attr->ah_flags & IB_AH_GRH) {
/* Header size in 32-bit words. */
@@ -368,11 +396,13 @@ int ipath_make_ud_req(struct ipath_qp *qp)
ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
done:
- if (++qp->s_cur >= qp->s_size)
- qp->s_cur = 0;
ret = 1;
+ goto unlock;
bail:
+ qp->s_flags &= ~IPATH_S_BUSY;
+unlock:
+ spin_unlock_irqrestore(&qp->s_lock, flags);
return ret;
}
@@ -506,8 +536,8 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
/*
* Get the next work request entry to find where to put the data.
*/
- if (qp->r_reuse_sge)
- qp->r_reuse_sge = 0;
+ if (qp->r_flags & IPATH_R_REUSE_SGE)
+ qp->r_flags &= ~IPATH_R_REUSE_SGE;
else if (!ipath_get_rwqe(qp, 0)) {
/*
* Count VL15 packets dropped due to no receive buffer.
@@ -523,7 +553,7 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
}
/* Silently drop packets which are too big. */
if (wc.byte_len > qp->r_len) {
- qp->r_reuse_sge = 1;
+ qp->r_flags |= IPATH_R_REUSE_SGE;
dev->n_pkt_drops++;
goto bail;
}
@@ -535,7 +565,8 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
ipath_skip_sge(&qp->r_sge, sizeof(struct ib_grh));
ipath_copy_sge(&qp->r_sge, data,
wc.byte_len - sizeof(struct ib_grh));
- qp->r_wrid_valid = 0;
+ if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))
+ goto bail;
wc.wr_id = qp->r_wr_id;
wc.status = IB_WC_SUCCESS;
wc.opcode = IB_WC_RECV;
diff --git a/drivers/infiniband/hw/ipath/ipath_user_sdma.c b/drivers/infiniband/hw/ipath/ipath_user_sdma.c
index 81da99b..fb26b03 100644
--- a/drivers/infiniband/hw/ipath/ipath_user_sdma.c
+++ b/drivers/infiniband/hw/ipath/ipath_user_sdma.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
diff --git a/drivers/infiniband/hw/ipath/ipath_user_sdma.h b/drivers/infiniband/hw/ipath/ipath_user_sdma.h
index c045af5..ce0448f 100644
--- a/drivers/infiniband/hw/ipath/ipath_user_sdma.h
+++ b/drivers/infiniband/hw/ipath/ipath_user_sdma.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.c b/drivers/infiniband/hw/ipath/ipath_verbs.c
index 28bd72f..7bdcc04 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.c
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -42,8 +42,10 @@
#ifdef __x86_64__
void *memcpy_cachebypass(void *, const void *, __kernel_size_t);
+void *memcpy_cachebypass2(void *, const void *, __kernel_size_t);
#else
#define memcpy_cachebypass(a,b,c) memcpy((a),(b),(c))
+#define memcpy_cachebypass2(a,b,c) memcpy((a),(b),(c))
#endif
static unsigned int ib_ipath_qp_table_size = 251;
@@ -117,16 +119,24 @@ static unsigned int ib_ipath_disable_sma;
module_param_named(disable_sma, ib_ipath_disable_sma, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(ib_ipath_disable_sma, "Disable the SMA");
+/*
+ * Note that it is OK to post send work requests in the SQE and ERR
+ * states; ipath_do_send() will process them and generate error
+ * completions as per IB 1.2 C10-96.
+ */
const int ib_ipath_state_ops[IB_QPS_ERR + 1] = {
[IB_QPS_RESET] = 0,
[IB_QPS_INIT] = IPATH_POST_RECV_OK,
[IB_QPS_RTR] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
[IB_QPS_RTS] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
- IPATH_POST_SEND_OK | IPATH_PROCESS_SEND_OK,
+ IPATH_POST_SEND_OK | IPATH_PROCESS_SEND_OK |
+ IPATH_PROCESS_NEXT_SEND_OK,
[IB_QPS_SQD] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
- IPATH_POST_SEND_OK,
- [IB_QPS_SQE] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
- [IB_QPS_ERR] = 0,
+ IPATH_POST_SEND_OK | IPATH_PROCESS_SEND_OK,
+ [IB_QPS_SQE] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
+ IPATH_POST_SEND_OK | IPATH_FLUSH_SEND,
+ [IB_QPS_ERR] = IPATH_POST_RECV_OK | IPATH_FLUSH_RECV |
+ IPATH_POST_SEND_OK | IPATH_FLUSH_SEND,
};
struct ipath_ucontext {
@@ -175,7 +185,10 @@ void ipath_copy_sge(struct ipath_sge_state *ss, void *data, u32 length)
if (len > sge->sge_length)
len = sge->sge_length;
BUG_ON(len == 0);
- memcpy_cachebypass(sge->vaddr, data, len);
+ if (ss->total_len > 512 * 1024)
+ memcpy_cachebypass2(sge->vaddr, data, len);
+ else
+ memcpy_cachebypass(sge->vaddr, data, len);
sge->vaddr += len;
sge->length -= len;
sge->sge_length -= len;
@@ -236,18 +249,6 @@ void ipath_skip_sge(struct ipath_sge_state *ss, u32 length)
}
}
-static void ipath_flush_wqe(struct ipath_qp *qp, struct ib_send_wr *wr)
-{
- struct ib_wc wc;
-
- memset(&wc, 0, sizeof(wc));
- wc.wr_id = wr->wr_id;
- wc.status = IB_WC_WR_FLUSH_ERR;
- wc.opcode = ib_ipath_wc_opcode[wr->opcode];
- wc.qp = &qp->ibqp;
- ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
-}
-
/*
* Count the number of DMA descriptors needed to send length bytes of data.
* Don't modify the ipath_sge_state to get the count.
@@ -353,14 +354,8 @@ static int ipath_post_one_send(struct ipath_qp *qp, struct ib_send_wr *wr)
spin_lock_irqsave(&qp->s_lock, flags);
/* Check that state is OK to post send. */
- if (unlikely(!(ib_ipath_state_ops[qp->state] & IPATH_POST_SEND_OK))) {
- if (qp->state != IB_QPS_SQE && qp->state != IB_QPS_ERR)
- goto bail_inval;
- /* C10-96 says generate a flushed completion entry. */
- ipath_flush_wqe(qp, wr);
- ret = 0;
- goto bail;
- }
+ if (unlikely(!(ib_ipath_state_ops[qp->state] & IPATH_POST_SEND_OK)))
+ goto bail_inval;
/* IB spec says that num_sge == 0 is OK. */
if (wr->num_sge > qp->s_max_sge)
@@ -402,7 +397,6 @@ static int ipath_post_one_send(struct ipath_qp *qp, struct ib_send_wr *wr)
wqe = get_swqe_ptr(qp, qp->s_head);
wqe->wr = *wr;
- wqe->ssn = qp->s_ssn++;
wqe->length = 0;
if (wr->num_sge) {
acc = wr->opcode >= IB_WR_RDMA_READ ?
@@ -428,6 +422,7 @@ static int ipath_post_one_send(struct ipath_qp *qp, struct ib_send_wr *wr)
goto bail_inval;
} else if (wqe->length > to_idev(qp->ibqp.device)->dd->ipath_ibmtu)
goto bail_inval;
+ wqe->ssn = qp->s_ssn++;
qp->s_head = next;
ret = 0;
@@ -683,6 +678,7 @@ bail:;
static void ipath_ib_timer(struct ipath_ibdev *dev)
{
struct ipath_qp *resend = NULL;
+ struct ipath_qp *rnr = NULL;
struct list_head *last;
struct ipath_qp *qp;
unsigned long flags;
@@ -709,7 +705,9 @@ static void ipath_ib_timer(struct ipath_ibdev *dev)
if (--qp->s_rnr_timeout == 0) {
do {
list_del_init(&qp->timerwait);
- ipath_schedule_send(qp);
+ qp->timer_next = rnr;
+ rnr = qp;
+ atomic_inc(&qp->refcount);
if (list_empty(last))
break;
qp = list_entry(last->next, struct ipath_qp,
@@ -749,13 +747,15 @@ static void ipath_ib_timer(struct ipath_ibdev *dev)
spin_unlock_irqrestore(&dev->pending_lock, flags);
/* XXX What if timer fires again while this is running? */
- for (qp = resend; qp != NULL; qp = qp->timer_next) {
- struct ib_wc wc;
+ while (resend != NULL) {
+ qp = resend;
+ resend = qp->timer_next;
spin_lock_irqsave(&qp->s_lock, flags);
- if (qp->s_last != qp->s_tail && qp->state == IB_QPS_RTS) {
+ if (qp->s_last != qp->s_tail &&
+ ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) {
dev->n_timeouts++;
- ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
+ ipath_restart_rc(qp, qp->s_last_psn + 1);
}
spin_unlock_irqrestore(&qp->s_lock, flags);
@@ -763,6 +763,19 @@ static void ipath_ib_timer(struct ipath_ibdev *dev)
if (atomic_dec_and_test(&qp->refcount))
wake_up(&qp->wait);
}
+ while (rnr != NULL) {
+ qp = rnr;
+ rnr = qp->timer_next;
+
+ spin_lock_irqsave(&qp->s_lock, flags);
+ if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)
+ ipath_schedule_send(qp);
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+
+ /* Notify ipath_destroy_qp() if it is waiting. */
+ if (atomic_dec_and_test(&qp->refcount))
+ wake_up(&qp->wait);
+ }
}
static void update_sge(struct ipath_sge_state *ss, u32 length)
@@ -1018,13 +1031,24 @@ static void sdma_complete(void *cookie, int status)
struct ipath_verbs_txreq *tx = cookie;
struct ipath_qp *qp = tx->qp;
struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+ unsigned int flags;
+ enum ib_wc_status ibs = status == IPATH_SDMA_TXREQ_S_OK ?
+ IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR;
- /* Generate a completion queue entry if needed */
- if (qp->ibqp.qp_type != IB_QPT_RC && tx->wqe) {
- enum ib_wc_status ibs = status == IPATH_SDMA_TXREQ_S_OK ?
- IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR;
-
+ if (atomic_dec_and_test(&qp->s_dma_busy)) {
+ spin_lock_irqsave(&qp->s_lock, flags);
+ if (tx->wqe)
+ ipath_send_complete(qp, tx->wqe, ibs);
+ if ((ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND &&
+ qp->s_last != qp->s_head) ||
+ (qp->s_flags & IPATH_S_WAIT_DMA))
+ ipath_schedule_send(qp);
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+ wake_up(&qp->wait_dma);
+ } else if (tx->wqe) {
+ spin_lock_irqsave(&qp->s_lock, flags);
ipath_send_complete(qp, tx->wqe, ibs);
+ spin_unlock_irqrestore(&qp->s_lock, flags);
}
if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEBUF)
@@ -1035,6 +1059,21 @@ static void sdma_complete(void *cookie, int status)
wake_up(&qp->wait);
}
+static void decrement_dma_busy(struct ipath_qp *qp)
+{
+ unsigned int flags;
+
+ if (atomic_dec_and_test(&qp->s_dma_busy)) {
+ spin_lock_irqsave(&qp->s_lock, flags);
+ if ((ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND &&
+ qp->s_last != qp->s_head) ||
+ (qp->s_flags & IPATH_S_WAIT_DMA))
+ ipath_schedule_send(qp);
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+ wake_up(&qp->wait_dma);
+ }
+}
+
/*
* Compute the number of clock cycles of delay before sending the next packet.
* The multipliers reflect the number of clocks for the fastest rate so
@@ -1073,9 +1112,12 @@ static int ipath_verbs_send_dma(struct ipath_qp *qp,
if (tx) {
qp->s_tx = NULL;
/* resend previously constructed packet */
+ atomic_inc(&qp->s_dma_busy);
ret = ipath_sdma_verbs_send(dd, tx->ss, tx->len, tx);
- if (ret)
+ if (ret) {
qp->s_tx = tx;
+ decrement_dma_busy(qp);
+ }
goto bail;
}
@@ -1126,12 +1168,14 @@ static int ipath_verbs_send_dma(struct ipath_qp *qp,
tx->txreq.sg_count = ndesc;
tx->map_len = (hdrwords + 2) << 2;
tx->txreq.map_addr = &tx->hdr;
+ atomic_inc(&qp->s_dma_busy);
ret = ipath_sdma_verbs_send(dd, ss, dwords, tx);
if (ret) {
/* save ss and length in dwords */
tx->ss = ss;
tx->len = dwords;
qp->s_tx = tx;
+ decrement_dma_busy(qp);
}
goto bail;
}
@@ -1152,6 +1196,7 @@ static int ipath_verbs_send_dma(struct ipath_qp *qp,
memcpy(piobuf, hdr, hdrwords << 2);
ipath_copy_from_sge(piobuf + hdrwords, ss, len);
+ atomic_inc(&qp->s_dma_busy);
ret = ipath_sdma_verbs_send(dd, NULL, 0, tx);
/*
* If we couldn't queue the DMA request, save the info
@@ -1162,6 +1207,7 @@ static int ipath_verbs_send_dma(struct ipath_qp *qp,
tx->ss = NULL;
tx->len = 0;
qp->s_tx = tx;
+ decrement_dma_busy(qp);
}
dev->n_unaligned++;
goto bail;
@@ -1185,6 +1231,7 @@ static int ipath_verbs_send_pio(struct ipath_qp *qp,
unsigned flush_wc;
u32 control;
int ret;
+ unsigned int flags;
piobuf = ipath_getpiobuf(dd, plen, NULL);
if (unlikely(piobuf == NULL)) {
@@ -1255,8 +1302,11 @@ static int ipath_verbs_send_pio(struct ipath_qp *qp,
}
copy_io(piobuf, ss, len, flush_wc);
done:
- if (qp->s_wqe)
+ if (qp->s_wqe) {
+ spin_lock_irqsave(&qp->s_lock, flags);
ipath_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS);
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+ }
ret = 0;
bail:
return ret;
@@ -1284,17 +1334,17 @@ int ipath_verbs_send(struct ipath_qp *qp, struct ipath_ib_header *hdr,
*/
plen = hdrwords + dwords + 1;
- /* Drop non-VL15 packets if we are not in the active state */
- if (!(dd->ipath_flags & IPATH_LINKACTIVE) &&
- qp->ibqp.qp_type != IB_QPT_SMI) {
- if (qp->s_wqe)
- ipath_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS);
- ret = 0;
- } else if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
- ret = ipath_verbs_send_dma(qp, hdr, hdrwords, ss, len,
+ /*
+ * VL15 packets (IB_QPT_SMI) will always use PIO, so we
+ * can defer SDMA restart until link goes ACTIVE without
+ * worrying about just how we got there.
+ */
+ if (qp->ibqp.qp_type == IB_QPT_SMI ||
+ !(dd->ipath_flags & IPATH_HAS_SEND_DMA))
+ ret = ipath_verbs_send_pio(qp, hdr, hdrwords, ss, len,
plen, dwords);
else
- ret = ipath_verbs_send_pio(qp, hdr, hdrwords, ss, len,
+ ret = ipath_verbs_send_dma(qp, hdr, hdrwords, ss, len,
plen, dwords);
return ret;
@@ -1407,22 +1457,41 @@ bail:
*/
int ipath_ib_piobufavail(struct ipath_ibdev *dev)
{
+ struct list_head *list;
+ struct ipath_qp *qplist;
struct ipath_qp *qp;
unsigned long flags;
if (dev == NULL)
goto bail;
+ list = &dev->piowait;
+ qplist = NULL;
+
spin_lock_irqsave(&dev->pending_lock, flags);
- while (!list_empty(&dev->piowait)) {
- qp = list_entry(dev->piowait.next, struct ipath_qp,
- piowait);
+ while (!list_empty(list)) {
+ qp = list_entry(list->next, struct ipath_qp, piowait);
list_del_init(&qp->piowait);
- clear_bit(IPATH_S_BUSY, &qp->s_busy);
- ipath_schedule_send(qp);
+ qp->pio_next = qplist;
+ qplist = qp;
+ atomic_inc(&qp->refcount);
}
spin_unlock_irqrestore(&dev->pending_lock, flags);
+ while (qplist != NULL) {
+ qp = qplist;
+ qplist = qp->pio_next;
+
+ spin_lock_irqsave(&qp->s_lock, flags);
+ if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)
+ ipath_schedule_send(qp);
+ spin_unlock_irqrestore(&qp->s_lock, flags);
+
+ /* Notify ipath_destroy_qp() if it is waiting. */
+ if (atomic_dec_and_test(&qp->refcount))
+ wake_up(&qp->wait);
+ }
+
bail:
return 0;
}
@@ -2145,11 +2214,12 @@ bail:
void ipath_unregister_ib_device(struct ipath_ibdev *dev)
{
struct ib_device *ibdev = &dev->ibdev;
-
- disable_timer(dev->dd);
+ u32 qps_inuse;
ib_unregister_device(ibdev);
+ disable_timer(dev->dd);
+
if (!list_empty(&dev->pending[0]) ||
!list_empty(&dev->pending[1]) ||
!list_empty(&dev->pending[2]))
@@ -2164,7 +2234,10 @@ void ipath_unregister_ib_device(struct ipath_ibdev *dev)
* Note that ipath_unregister_ib_device() can be called before all
* the QPs are destroyed!
*/
- ipath_free_all_qps(&dev->qp_table);
+ qps_inuse = ipath_free_all_qps(&dev->qp_table);
+ if (qps_inuse)
+ ipath_dev_err(dev->dd, "QP memory leak! %u still in use\n",
+ qps_inuse);
kfree(dev->qp_table.table);
kfree(dev->lk_table.table);
kfree(dev->txreq_bufs);
@@ -2212,17 +2285,14 @@ static ssize_t show_stats(struct class_device *cdev, char *buf)
"RC OTH NAKs %d\n"
"RC timeouts %d\n"
"RC RDMA dup %d\n"
- "RC stalls %d\n"
"piobuf wait %d\n"
- "no piobuf %d\n"
"unaligned %d\n"
"PKT drops %d\n"
"WQE errs %d\n",
dev->n_rc_resends, dev->n_rc_qacks, dev->n_rc_acks,
dev->n_seq_naks, dev->n_rdma_seq, dev->n_rnr_naks,
dev->n_other_naks, dev->n_timeouts,
- dev->n_rdma_dup_busy, dev->n_rc_stalls, dev->n_piowait,
- dev->n_no_piobuf, dev->n_unaligned,
+ dev->n_rdma_dup_busy, dev->n_piowait, dev->n_unaligned,
dev->n_pkt_drops, dev->n_wqe_errs);
for (i = 0; i < ARRAY_SIZE(dev->opstats); i++) {
const struct ipath_opcode_stats *si = &dev->opstats[i];
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.h b/drivers/infiniband/hw/ipath/ipath_verbs.h
index 4aade31..9b21282 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.h
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -74,6 +74,11 @@
#define IPATH_POST_RECV_OK 0x02
#define IPATH_PROCESS_RECV_OK 0x04
#define IPATH_PROCESS_SEND_OK 0x08
+#define IPATH_PROCESS_NEXT_SEND_OK 0x10
+#define IPATH_FLUSH_SEND 0x20
+#define IPATH_FLUSH_RECV 0x40
+#define IPATH_PROCESS_OR_FLUSH_SEND \
+ (IPATH_PROCESS_SEND_OK | IPATH_FLUSH_SEND)
/* IB Performance Manager status values */
#define IB_PMA_SAMPLE_STATUS_DONE 0x00
@@ -323,6 +328,7 @@ struct ipath_srq {
struct ipath_sge_state {
struct ipath_sge *sg_list; /* next SGE to be used if any */
struct ipath_sge sge; /* progress state for the current SGE */
+ u32 total_len;
u8 num_sge;
u8 static_rate;
};
@@ -353,12 +359,14 @@ struct ipath_qp {
struct ib_qp ibqp;
struct ipath_qp *next; /* link list for QPN hash table */
struct ipath_qp *timer_next; /* link list for ipath_ib_timer() */
+ struct ipath_qp *pio_next; /* link for ipath_ib_piobufavail() */
struct list_head piowait; /* link for wait PIO buf */
struct list_head timerwait; /* link for waiting for timeouts */
struct ib_ah_attr remote_ah_attr;
struct ipath_ib_header s_hdr; /* next packet header to send */
atomic_t refcount;
wait_queue_head_t wait;
+ wait_queue_head_t wait_dma;
struct tasklet_struct s_task;
struct ipath_mmap_info *ip;
struct ipath_sge_state *s_cur_sge;
@@ -369,7 +377,7 @@ struct ipath_qp {
struct ipath_sge_state s_rdma_read_sge;
struct ipath_sge_state r_sge; /* current receive data */
spinlock_t s_lock;
- unsigned long s_busy;
+ atomic_t s_dma_busy;
u16 s_pkt_delay;
u16 s_hdrwords; /* size of s_hdr in 32 bit words */
u32 s_cur_size; /* size of send packet in bytes */
@@ -383,6 +391,7 @@ struct ipath_qp {
u32 s_rnr_timeout; /* number of milliseconds for RNR timeout */
u32 r_ack_psn; /* PSN for next ACK or atomic ACK */
u64 r_wr_id; /* ID for current receive WQE */
+ unsigned long r_aflags;
u32 r_len; /* total length of r_sge */
u32 r_rcv_len; /* receive data len processed */
u32 r_psn; /* expected rcv packet sequence number */
@@ -394,8 +403,7 @@ struct ipath_qp {
u8 r_state; /* opcode of last packet received */
u8 r_nak_state; /* non-zero if NAK is pending */
u8 r_min_rnr_timer; /* retry timeout value for RNR NAKs */
- u8 r_reuse_sge; /* for UC receive errors */
- u8 r_wrid_valid; /* r_wrid set but CQ entry not yet made */
+ u8 r_flags;
u8 r_max_rd_atomic; /* max number of RDMA read/atomic to receive */
u8 r_head_ack_queue; /* index into s_ack_queue[] */
u8 qp_access_flags;
@@ -404,13 +412,13 @@ struct ipath_qp {
u8 s_rnr_retry_cnt;
u8 s_retry; /* requester retry counter */
u8 s_rnr_retry; /* requester RNR retry counter */
- u8 s_wait_credit; /* limit number of unacked packets sent */
u8 s_pkey_index; /* PKEY index to use */
u8 s_max_rd_atomic; /* max number of RDMA read/atomic to send */
u8 s_num_rd_atomic; /* number of RDMA read/atomic pending */
u8 s_tail_ack_queue; /* index into s_ack_queue[] */
u8 s_flags;
u8 s_dmult;
+ u8 s_draining;
u8 timeout; /* Timeout for this QP */
enum ib_mtu path_mtu;
u32 remote_qpn;
@@ -428,17 +436,40 @@ struct ipath_qp {
struct ipath_sge r_sg_list[0]; /* verified SGEs */
};
-/* Bit definition for s_busy. */
-#define IPATH_S_BUSY 0
-#define IPATH_S_DESTROYING 1
+/*
+ * Atomic bit definitions for r_aflags.
+ */
+#define IPATH_R_WRID_VALID 0
+
+/*
+ * Bit definitions for r_flags.
+ */
+#define IPATH_R_REUSE_SGE 0x01
+#define IPATH_R_RDMAR_SEQ 0x02
/*
* Bit definitions for s_flags.
+ *
+ * IPATH_S_FENCE_PENDING - waiting for all prior RDMA read or atomic SWQEs
+ * before processing the next SWQE
+ * IPATH_S_RDMAR_PENDING - waiting for any RDMA read or atomic SWQEs
+ * before processing the next SWQE
+ * IPATH_S_WAITING - waiting for RNR timeout or send buffer available.
+ * IPATH_S_WAIT_SSN_CREDIT - waiting for RC credits to process next SWQE
+ * IPATH_S_WAIT_DMA - waiting for send DMA queue to drain before generating
+ next send completion entry not via send DMA.
*/
#define IPATH_S_SIGNAL_REQ_WR 0x01
#define IPATH_S_FENCE_PENDING 0x02
#define IPATH_S_RDMAR_PENDING 0x04
#define IPATH_S_ACK_PENDING 0x08
+#define IPATH_S_BUSY 0x10
+#define IPATH_S_WAITING 0x20
+#define IPATH_S_WAIT_SSN_CREDIT 0x40
+#define IPATH_S_WAIT_DMA 0x80
+
+#define IPATH_S_ANY_WAIT (IPATH_S_FENCE_PENDING | IPATH_S_RDMAR_PENDING | \
+ IPATH_S_WAITING | IPATH_S_WAIT_SSN_CREDIT | IPATH_S_WAIT_DMA)
#define IPATH_PSN_CREDIT 512
@@ -574,13 +605,11 @@ struct ipath_ibdev {
u32 n_rnr_naks;
u32 n_other_naks;
u32 n_timeouts;
- u32 n_rc_stalls;
u32 n_pkt_drops;
u32 n_vl15_dropped;
u32 n_wqe_errs;
u32 n_rdma_dup_busy;
u32 n_piowait;
- u32 n_no_piobuf;
u32 n_unaligned;
u32 port_cap_flags;
u32 pma_sample_start;
@@ -658,9 +687,14 @@ static inline struct ipath_ibdev *to_idev(struct ib_device *ibdev)
return container_of(ibdev, struct ipath_ibdev, ibdev);
}
+/*
+ * This must be called with s_lock held.
+ */
static inline void ipath_schedule_send(struct ipath_qp *qp)
{
- if (!test_bit(IPATH_S_DESTROYING, &qp->s_busy))
+ if (qp->s_flags & IPATH_S_ANY_WAIT)
+ qp->s_flags &= ~IPATH_S_ANY_WAIT;
+ if (!(qp->s_flags & IPATH_S_BUSY))
tasklet_hi_schedule(&qp->s_task);
}
@@ -713,12 +747,10 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_qp_init_attr *init_attr);
-void ipath_free_all_qps(struct ipath_qp_table *qpt);
+unsigned ipath_free_all_qps(struct ipath_qp_table *qpt);
int ipath_init_qp_table(struct ipath_ibdev *idev, int size);
-void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc);
-
void ipath_get_credit(struct ipath_qp *qp, u32 aeth);
unsigned ipath_ib_rate_to_mult(enum ib_rate rate);
@@ -738,7 +770,9 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
int has_grh, void *data, u32 tlen, struct ipath_qp *qp);
-void ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc);
+void ipath_restart_rc(struct ipath_qp *qp, u32 psn);
+
+void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err);
int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr);
diff --git a/drivers/infiniband/hw/ipath/memcpy_cachebypass_x86_64.S b/drivers/infiniband/hw/ipath/memcpy_cachebypass_x86_64.S
index 9e01a4c..cbdedee 100644
--- a/drivers/infiniband/hw/ipath/memcpy_cachebypass_x86_64.S
+++ b/drivers/infiniband/hw/ipath/memcpy_cachebypass_x86_64.S
@@ -1,8 +1,42 @@
+/*
+ * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
.text
.p2align 4,,15
/* rdi destination, rsi source, rdx count */
.globl memcpy_cachebypass
.type memcpy_cachebypass, @function
+# loads bypass the cache, stores fill the cache
memcpy_cachebypass:
movq %rdi, %rax
.L5:
@@ -59,7 +93,7 @@ memcpy_cachebypass:
.L64:
movl $64, %ecx
.L42:
- prefetchnta 128(%rsi)
+ prefetchnta 256(%rsi)
movq (%rsi), %r8
movq 8(%rsi), %r9
movq 16(%rsi), %r10
@@ -113,3 +147,120 @@ memcpy_cachebypass:
movq (%rsi), %rcx
movq %rcx, (%rdi)
ret
+
+ .text
+ .p2align 4,,15
+ /* rdi destination, rsi source, rdx count */
+ .globl memcpy_cachebypass2
+ .type memcpy_cachebypass2, @function
+# both loads and stores bypass the cache
+memcpy_cachebypass2:
+ movq %rdi, %rax
+.L2_5:
+ cmpq $15, %rdx
+ ja .L2_34
+.L2_3:
+ cmpl $8, %edx /* rdx is 0..15 */
+ jbe .L2_9
+.L2_6:
+ testb $8, %dxl /* rdx is 3,5,6,7,9..15 */
+ je .L2_13
+ movq (%rsi), %rcx
+ addq $8, %rsi
+ movq %rcx, (%rdi)
+ addq $8, %rdi
+.L2_13:
+ testb $4, %dxl
+ je .L2_15
+ movl (%rsi), %ecx
+ addq $4, %rsi
+ movl %ecx, (%rdi)
+ addq $4, %rdi
+.L2_15:
+ testb $2, %dxl
+ je .L2_17
+ movzwl (%rsi), %ecx
+ addq $2, %rsi
+ movw %cx, (%rdi)
+ addq $2, %rdi
+.L2_17:
+ testb $1, %dxl
+ je .L2_33
+.L2_1:
+ movzbl (%rsi), %ecx
+ movb %cl, (%rdi)
+.L2_33:
+ ret
+.L2_34:
+ cmpq $63, %rdx /* rdx is > 15 */
+ ja .L2_64
+ movl $16, %ecx /* rdx is 16..63 */
+.L2_25:
+ movq 8(%rsi), %r8
+ movq (%rsi), %r9
+ addq %rcx, %rsi
+ movq %r8, 8(%rdi)
+ movq %r9, (%rdi)
+ addq %rcx, %rdi
+ subq %rcx, %rdx
+ cmpl %edx, %ecx /* is rdx >= 16? */
+ jbe .L2_25
+ jmp .L2_3 /* rdx is 0..15 */
+ .p2align 4,,7
+.L2_64:
+ movl $64, %ecx
+.L2_42:
+ prefetchnta 256(%rsi)
+ movq (%rsi), %r8
+ movq 8(%rsi), %r9
+ movq 16(%rsi), %r10
+ movq 24(%rsi), %r11
+ subq %rcx, %rdx
+ movnti %r8, (%rdi)
+ movq 32(%rsi), %r8
+ movnti %r9, 8(%rdi)
+ movq 40(%rsi), %r9
+ movnti %r10, 16(%rdi)
+ movq 48(%rsi), %r10
+ movnti %r11, 24(%rdi)
+ movq 56(%rsi), %r11
+ addq %rcx, %rsi
+ movnti %r8, 32(%rdi)
+ movnti %r9, 40(%rdi)
+ movnti %r10, 48(%rdi)
+ movnti %r11, 56(%rdi)
+ addq %rcx, %rdi
+ cmpq %rdx, %rcx /* is rdx >= 64? */
+ jbe .L2_42
+ sfence
+ orl %edx, %edx
+ je .L2_33
+ jmp .L2_5
+.L2_9:
+ jmp *.L2_12(,%rdx,8) /* rdx is 0..8 */
+ .section .rodata
+ .align 8
+ .align 4
+.L2_12:
+ .quad .L2_33
+ .quad .L2_1
+ .quad .L2_2
+ .quad .L2_6
+ .quad .L2_4
+ .quad .L2_6
+ .quad .L2_6
+ .quad .L2_6
+ .quad .L2_8
+ .text
+.L2_2:
+ movzwl (%rsi), %ecx
+ movw %cx, (%rdi)
+ ret
+.L2_4:
+ movl (%rsi), %ecx
+ movl %ecx, (%rdi)
+ ret
+.L2_8:
+ movq (%rsi), %rcx
+ movq %rcx, (%rdi)
+ ret
diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index 0ed02b7..ee72ebe 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -255,7 +255,8 @@ int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
return IB_MAD_RESULT_SUCCESS;
} else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT ||
in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS1 ||
- in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS2) {
+ in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS2 ||
+ in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_CONG_MGMT) {
if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET &&
in_mad->mad_hdr.method != IB_MGMT_METHOD_SET)
return IB_MAD_RESULT_SUCCESS;
diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c
index 252db08..0b92d3e 100644
--- a/drivers/infiniband/hw/mthca/mthca_memfree.c
+++ b/drivers/infiniband/hw/mthca/mthca_memfree.c
@@ -109,7 +109,11 @@ static int mthca_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_m
{
struct page *page;
- page = alloc_pages(gfp_mask, order);
+ /*
+ * Use __GFP_ZERO because buggy firmware assumes ICM pages are
+ * cleared, and subtle failures are seen if they aren't.
+ */
+ page = alloc_pages(gfp_mask | __GFP_ZERO, order);
if (!page)
return -ENOMEM;
diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/infiniband/hw/mthca/mthca_mr.c
index 3538da1..820205d 100644
--- a/drivers/infiniband/hw/mthca/mthca_mr.c
+++ b/drivers/infiniband/hw/mthca/mthca_mr.c
@@ -818,15 +818,9 @@ int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
{
- u32 key;
-
if (!fmr->maps)
return;
- key = tavor_key_to_hw_index(fmr->ibmr.lkey);
- key &= dev->limits.num_mpts - 1;
- fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
-
fmr->maps = 0;
writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
@@ -834,16 +828,9 @@ void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
{
- u32 key;
-
if (!fmr->maps)
return;
- key = arbel_key_to_hw_index(fmr->ibmr.lkey);
- key &= dev->limits.num_mpts - 1;
- key = adjust_key(dev, key);
- fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
-
fmr->maps = 0;
*(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 067fbd7..1296b61 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -249,7 +249,7 @@ void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
spin_unlock(&dev->qp_table.lock);
if (!qp) {
- mthca_dbg(dev, "Async event %d for bogus QP %08x\n",
+ mthca_warn(dev, "Async event %d for bogus QP %08x\n",
(int) event_type, qpn);
return;
}
diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c
index cbd1347..4da5ad7 100644
--- a/drivers/infiniband/hw/nes/nes.c
+++ b/drivers/infiniband/hw/nes/nes.c
@@ -751,13 +751,13 @@ static void __devexit nes_remove(struct pci_dev *pcidev)
list_del(&nesdev->list);
nes_destroy_cqp(nesdev);
+
+ free_irq(pcidev->irq, nesdev);
tasklet_kill(&nesdev->dpc_tasklet);
/* Deallocate the Adapter Structure */
nes_destroy_adapter(nesdev->nesadapter);
- free_irq(pcidev->irq, nesdev);
-
if (nesdev->msi_enabled) {
pci_disable_msi(pcidev);
}
diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h
index 81d2eca..6be3446 100644
--- a/drivers/infiniband/hw/nes/nes.h
+++ b/drivers/infiniband/hw/nes/nes.h
@@ -61,7 +61,7 @@
#define DRV_BUILD "1"
#define DRV_NAME "iw_nes"
-#define DRV_VERSION "1.0 OFED Build " DRV_BUILD
+#define DRV_VERSION "1.3.1 OFED Build " DRV_BUILD
#define PFX DRV_NAME ": "
/*
@@ -562,8 +562,8 @@ void nes_unregister_ofa_device(struct nes_ib_device *);
int nes_read_eeprom_values(struct nes_device *, struct nes_adapter *);
void nes_write_1G_phy_reg(struct nes_device *, u8, u8, u16);
void nes_read_1G_phy_reg(struct nes_device *, u8, u8, u16 *);
-void nes_write_10G_phy_reg(struct nes_device *, u16, u8, u16);
-void nes_read_10G_phy_reg(struct nes_device *, u16, u8);
+void nes_write_10G_phy_reg(struct nes_device *, u16, u8, u16, u16);
+void nes_read_10G_phy_reg(struct nes_device *, u8, u8, u16);
struct nes_cqp_request *nes_get_cqp_request(struct nes_device *);
void nes_post_cqp_request(struct nes_device *, struct nes_cqp_request *, int);
int nes_arp_table(struct nes_device *, u32, u8 *, u32);
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index 4705dbc..d317aeb 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -40,6 +40,7 @@
#include <linux/tcp.h>
#include <linux/init.h>
#include <linux/if_arp.h>
+#include <linux/if_vlan.h>
#include <linux/notifier.h>
#include <linux/net.h>
#include <linux/types.h>
@@ -394,7 +395,7 @@ int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb,
}
if (type == NES_TIMER_TYPE_SEND) {
- new_send->seq_num = htonl(tcp_hdr(skb)->seq);
+ new_send->seq_num = ntohl(tcp_hdr(skb)->seq);
atomic_inc(&new_send->skb->users);
ret = nes_nic_cm_xmit(new_send->skb, cm_node->netdev);
@@ -419,7 +420,7 @@ int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb,
spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
}
if (type == NES_TIMER_TYPE_RECV) {
- new_send->seq_num = htonl(tcp_hdr(skb)->seq);
+ new_send->seq_num = ntohl(tcp_hdr(skb)->seq);
new_send->timetosend = jiffies;
spin_lock_irqsave(&cm_node->recv_list_lock, flags);
list_add_tail(&new_send->list, &cm_node->recv_list);
@@ -1072,7 +1073,7 @@ static struct nes_cm_node *make_cm_node(struct nes_cm_core *cm_core,
ts = current_kernel_time();
cm_node->tcp_cntxt.loc_seq_num = htonl(ts.tv_nsec);
cm_node->tcp_cntxt.mss = nesvnic->max_frame_size - sizeof(struct iphdr) -
- sizeof(struct tcphdr) - ETH_HLEN;
+ sizeof(struct tcphdr) - ETH_HLEN - VLAN_HLEN;
cm_node->tcp_cntxt.rcv_nxt = 0;
/* get a unique session ID , add thread_id to an upcounter to handle race */
atomic_inc(&cm_core->node_cnt);
@@ -1833,8 +1834,10 @@ int mini_cm_recv_pkt(struct nes_cm_core *cm_core, struct nes_vnic *nesvnic,
nfo.rem_addr = ntohl(iph->saddr);
nfo.rem_port = ntohs(tcph->source);
- nes_debug(NES_DBG_CM, "Received packet: dest=0x%08X:0x%04X src=0x%08X:0x%04X\n",
- iph->daddr, tcph->dest, iph->saddr, tcph->source);
+ nes_debug(NES_DBG_CM, "Received packet: dest=" NIPQUAD_FMT
+ ":0x%04X src=" NIPQUAD_FMT ":0x%04X\n",
+ NIPQUAD(iph->daddr), tcph->dest,
+ NIPQUAD(iph->saddr), tcph->source);
/* note: this call is going to increment cm_node ref count */
cm_node = find_node(cm_core,
diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c
index 404c343..ac6dfa6 100644
--- a/drivers/infiniband/hw/nes/nes_hw.c
+++ b/drivers/infiniband/hw/nes/nes_hw.c
@@ -624,6 +624,15 @@ unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
return 0;
}
+
+ i = 0;
+ while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
+ mdelay(1);
+ if (i >= 10000) {
+ printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
+ nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
+ return 0;
+ }
}
/* port reset */
@@ -672,17 +681,6 @@ unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
}
}
-
-
- i = 0;
- while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
- mdelay(1);
- if (i >= 10000) {
- printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
- nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
- return 0;
- }
-
return port_count;
}
@@ -1196,11 +1194,16 @@ int nes_init_phy(struct nes_device *nesdev)
{
struct nes_adapter *nesadapter = nesdev->nesadapter;
u32 counter = 0;
+ u32 sds_common_control0;
u32 mac_index = nesdev->mac_index;
- u32 tx_config;
+ u32 tx_config = 0;
u16 phy_data;
+ u32 temp_phy_data = 0;
+ u32 temp_phy_data2 = 0;
+ u32 i = 0;
- if (nesadapter->OneG_Mode) {
+ if ((nesadapter->OneG_Mode) &&
+ (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
printk(PFX "%s: Programming mdc config for 1G\n", __FUNCTION__);
@@ -1266,12 +1269,126 @@ int nes_init_phy(struct nes_device *nesdev)
nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
} else {
- if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
+ if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
+ (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
/* setup 10G MDIO operation */
tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
tx_config |= 0x14;
nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
}
+ if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
+
+ temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ mdelay(10);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
+ temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+
+ /*
+ * if firmware is already running (like from a
+ * driver un-load/load, don't do anything.
+ */
+ if (temp_phy_data == temp_phy_data2) {
+ /* configure QT2505 AMCC PHY */
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0000);
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
+
+ /*
+ * remove micro from reset; chip boots from ROM,
+ * uploads EEPROM f/w image, uC executes f/w
+ */
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
+
+ /*
+ * wait for heart beat to start to
+ * know loading is done
+ */
+ counter = 0;
+ do {
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
+ temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ if (counter++ > 1000) {
+ nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
+ break;
+ }
+ mdelay(100);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
+ temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ } while ((temp_phy_data2 == temp_phy_data));
+
+ /*
+ * wait for tracking to start to know
+ * f/w is good to go
+ */
+ counter = 0;
+ do {
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
+ temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ if (counter++ > 1000) {
+ nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
+ break;
+ }
+ mdelay(1000);
+ /*
+ * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
+ * temp_phy_data);
+ */
+ } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
+
+ /* set LOS Control invert RXLOSB_I_PADINV */
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
+ /* set LOS Control to mask of RXLOSB_I */
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
+ /* set LED1 to input mode (LED1 and LED2 share same LED) */
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
+ /* set LED2 to RX link_status and activity */
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
+ /* set LED3 to RX link_status */
+ nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
+
+ /*
+ * reset the res-calibration on t2
+ * serdes; ensures it is stable after
+ * the amcc phy is stable
+ */
+
+ sds_common_control0 = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
+ sds_common_control0 |= 0x1;
+ nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
+
+ /* release the res-calibration reset */
+ sds_common_control0 &= 0xfffffffe;
+ nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
+
+ i = 0;
+ while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
+ && (i++ < 5000)) {
+ /* mdelay(1); */
+ }
+
+ /*
+ * wait for link train done before moving on,
+ * or will get an interupt storm
+ */
+ counter = 0;
+ do {
+ temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
+ (0x200 * (nesdev->mac_index & 1)));
+ if (counter++ > 1000) {
+ nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
+ break;
+ }
+ mdelay(1);
+ } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
+ }
+ }
}
return 0;
}
@@ -2072,6 +2189,8 @@ void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
u32 u32temp;
u16 phy_data;
u16 temp_phy_data;
+ u32 pcs_val = 0x0f0f0000;
+ u32 pcs_mask = 0x0f1f0000;
spin_lock_irqsave(&nesadapter->phy_lock, flags);
if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
@@ -2135,13 +2254,30 @@ void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
- pcs_control_status = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
- pcs_control_status = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
+
+ if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
+ switch (mac_index) {
+ case 1:
+ case 3:
+ pcs_control_status = nes_read_indexed(nesdev,
+ NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
+ break;
+ default:
+ pcs_control_status = nes_read_indexed(nesdev,
+ NES_IDX_PHY_PCS_CONTROL_STATUS0);
+ break;
+ }
+ } else {
+ pcs_control_status = nes_read_indexed(nesdev,
+ NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
+ pcs_control_status = nes_read_indexed(nesdev,
+ NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
+ }
+
nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
mac_index, pcs_control_status);
- if (nesadapter->OneG_Mode) {
+ if ((nesadapter->OneG_Mode) &&
+ (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
u32temp = 0x01010000;
if (nesadapter->port_count > 2) {
u32temp |= 0x02020000;
@@ -2150,24 +2286,59 @@ void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
phy_data = 0;
nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
}
- } else if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
- nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
- temp_phy_data = (u16)nes_read_indexed(nesdev,
- NES_IDX_MAC_MDIO_CONTROL);
- u32temp = 20;
- do {
- nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
- phy_data = (u16)nes_read_indexed(nesdev,
- NES_IDX_MAC_MDIO_CONTROL);
- if ((phy_data == temp_phy_data) || (!(--u32temp)))
- break;
- temp_phy_data = phy_data;
- } while (1);
- nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
- __FUNCTION__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
-
} else {
- phy_data = (0x0f0f0000 == (pcs_control_status & 0x0f1f0000)) ? 4 : 0;
+ switch (nesadapter->phy_type[mac_index]) {
+ case NES_PHY_TYPE_IRIS:
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
+ temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ u32temp = 20;
+ do {
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
+ phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ if ((phy_data == temp_phy_data) || (!(--u32temp)))
+ break;
+ temp_phy_data = phy_data;
+ } while (1);
+ nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
+ __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
+ break;
+
+ case NES_PHY_TYPE_ARGUS:
+ /* clear the alarms */
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
+ /* check link status */
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
+ temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ u32temp = 100;
+ do {
+ nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
+
+ phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
+ if ((phy_data == temp_phy_data) || (!(--u32temp)))
+ break;
+ temp_phy_data = phy_data;
+ } while (1);
+ nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
+ __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
+ break;
+
+ case NES_PHY_TYPE_PUMA_1G:
+ if (mac_index < 2)
+ pcs_val = pcs_mask = 0x01010000;
+ else
+ pcs_val = pcs_mask = 0x02020000;
+ /* fall through */
+ default:
+ phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
+ break;
+ }
}
if (phy_data & 0x0004) {
@@ -2176,8 +2347,8 @@ void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n",
nesvnic->linkup);
if (nesvnic->linkup == 0) {
- printk(PFX "The Link is now up for port %u, netdev %p.\n",
- mac_index, nesvnic->netdev);
+ printk(PFX "The Link is now up for port %s, netdev %p.\n",
+ nesvnic->netdev->name, nesvnic->netdev);
if (netif_queue_stopped(nesvnic->netdev))
netif_start_queue(nesvnic->netdev);
nesvnic->linkup = 1;
@@ -2190,8 +2361,8 @@ void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
nesvnic->linkup);
if (nesvnic->linkup == 1) {
- printk(PFX "The Link is now down for port %u, netdev %p.\n",
- mac_index, nesvnic->netdev);
+ printk(PFX "The Link is now down for port %s, netdev %p.\n",
+ nesvnic->netdev->name, nesvnic->netdev);
if (!(netif_queue_stopped(nesvnic->netdev)))
netif_stop_queue(nesvnic->netdev);
nesvnic->linkup = 0;
diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h
index 8f784bc..58c84cb 100644
--- a/drivers/infiniband/hw/nes/nes_hw.h
+++ b/drivers/infiniband/hw/nes/nes_hw.h
@@ -33,8 +33,10 @@
#ifndef __NES_HW_H
#define __NES_HW_H
-#define NES_PHY_TYPE_1G 2
-#define NES_PHY_TYPE_IRIS 3
+#define NES_PHY_TYPE_1G 2
+#define NES_PHY_TYPE_IRIS 3
+#define NES_PHY_TYPE_ARGUS 4
+#define NES_PHY_TYPE_PUMA_1G 5
#define NES_PHY_TYPE_PUMA_10G 6
#define NES_MULTICAST_PF_MAX 8
@@ -905,7 +907,7 @@ struct nes_hw_qp {
};
struct nes_hw_cq {
- struct nes_hw_cqe volatile *cq_vbase; /* PCI memory for host rings */
+ struct nes_hw_cqe *cq_vbase; /* PCI memory for host rings */
void (*ce_handler)(struct nes_device *nesdev, struct nes_hw_cq *cq);
dma_addr_t cq_pbase; /* PCI memory for host rings */
u16 cq_head;
diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c
index d78cc76..1e755ba 100644
--- a/drivers/infiniband/hw/nes/nes_nic.c
+++ b/drivers/infiniband/hw/nes/nes_nic.c
@@ -1402,21 +1402,29 @@ static int nes_netdev_get_settings(struct net_device *netdev, struct ethtool_cmd
et_cmd->duplex = DUPLEX_FULL;
et_cmd->port = PORT_MII;
+
if (nesadapter->OneG_Mode) {
- et_cmd->supported = SUPPORTED_1000baseT_Full|SUPPORTED_Autoneg;
- et_cmd->advertising = ADVERTISED_1000baseT_Full|ADVERTISED_Autoneg;
et_cmd->speed = SPEED_1000;
- nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[nesdev->mac_index],
- &phy_data);
- if (phy_data&0x1000) {
- et_cmd->autoneg = AUTONEG_ENABLE;
+ if (nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_PUMA_1G) {
+ et_cmd->supported = SUPPORTED_1000baseT_Full;
+ et_cmd->advertising = ADVERTISED_1000baseT_Full;
+ et_cmd->autoneg = AUTONEG_DISABLE;
+ et_cmd->transceiver = XCVR_INTERNAL;
+ et_cmd->phy_address = nesdev->mac_index;
} else {
- et_cmd->autoneg = AUTONEG_DISABLE;
+ et_cmd->supported = SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg;
+ et_cmd->advertising = ADVERTISED_1000baseT_Full | ADVERTISED_Autoneg;
+ nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[nesdev->mac_index], &phy_data);
+ if (phy_data & 0x1000)
+ et_cmd->autoneg = AUTONEG_ENABLE;
+ else
+ et_cmd->autoneg = AUTONEG_DISABLE;
+ et_cmd->transceiver = XCVR_EXTERNAL;
+ et_cmd->phy_address = nesadapter->phy_index[nesdev->mac_index];
}
- et_cmd->transceiver = XCVR_EXTERNAL;
- et_cmd->phy_address = nesadapter->phy_index[nesdev->mac_index];
} else {
- if (nesadapter->phy_type[nesvnic->logical_port] == NES_PHY_TYPE_IRIS) {
+ if ((nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_IRIS) ||
+ (nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_ARGUS)) {
et_cmd->transceiver = XCVR_EXTERNAL;
et_cmd->port = PORT_FIBRE;
et_cmd->supported = SUPPORTED_FIBRE;
@@ -1447,7 +1455,8 @@ static int nes_netdev_set_settings(struct net_device *netdev, struct ethtool_cmd
struct nes_adapter *nesadapter = nesdev->nesadapter;
u16 phy_data;
- if (nesadapter->OneG_Mode) {
+ if ((nesadapter->OneG_Mode) &&
+ (nesadapter->phy_type[nesdev->mac_index] != NES_PHY_TYPE_PUMA_1G)) {
nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[nesdev->mac_index],
&phy_data);
if (et_cmd->autoneg) {
@@ -1638,27 +1647,34 @@ struct net_device *nes_netdev_init(struct nes_device *nesdev,
list_add_tail(&nesvnic->list, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]);
if ((nesdev->netdev_count == 0) &&
- (PCI_FUNC(nesdev->pcidev->devfn) == nesdev->mac_index)) {
- nes_debug(NES_DBG_INIT, "Setting up PHY interrupt mask. Using register index 0x%04X\n",
- NES_IDX_PHY_PCS_CONTROL_STATUS0+(0x200*(nesvnic->logical_port&1)));
+ ((PCI_FUNC(nesdev->pcidev->devfn) == nesdev->mac_index) ||
+ ((nesdev->nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_PUMA_1G) &&
+ (((PCI_FUNC(nesdev->pcidev->devfn) == 1) && (nesdev->mac_index == 2)) ||
+ ((PCI_FUNC(nesdev->pcidev->devfn) == 2) && (nesdev->mac_index == 1)))))) {
+ /*
+ * nes_debug(NES_DBG_INIT, "Setting up PHY interrupt mask. Using register index 0x%04X\n",
+ * NES_IDX_PHY_PCS_CONTROL_STATUS0 + (0x200 * (nesvnic->logical_port & 1)));
+ */
u32temp = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
- (0x200*(nesvnic->logical_port&1)));
- u32temp |= 0x00200000;
- nes_write_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
- (0x200*(nesvnic->logical_port&1)), u32temp);
+ (0x200 * (nesdev->mac_index & 1)));
+ if (nesdev->nesadapter->phy_type[nesdev->mac_index] != NES_PHY_TYPE_PUMA_1G) {
+ u32temp |= 0x00200000;
+ nes_write_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
+ (0x200 * (nesdev->mac_index & 1)), u32temp);
+ }
+
u32temp = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
- (0x200*(nesvnic->logical_port&1)) );
+ (0x200 * (nesdev->mac_index & 1)));
+
if ((u32temp&0x0f1f0000) == 0x0f0f0000) {
- if (nesdev->nesadapter->phy_type[nesvnic->logical_port] == NES_PHY_TYPE_IRIS) {
+ if (nesdev->nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_IRIS) {
nes_init_phy(nesdev);
- nes_read_10G_phy_reg(nesdev, 1,
- nesdev->nesadapter->phy_index[nesvnic->logical_port]);
+ nes_read_10G_phy_reg(nesdev, nesdev->nesadapter->phy_index[nesdev->mac_index], 1, 1);
temp_phy_data = (u16)nes_read_indexed(nesdev,
NES_IDX_MAC_MDIO_CONTROL);
u32temp = 20;
do {
- nes_read_10G_phy_reg(nesdev, 1,
- nesdev->nesadapter->phy_index[nesvnic->logical_port]);
+ nes_read_10G_phy_reg(nesdev, nesdev->nesadapter->phy_index[nesdev->mac_index], 1, 1);
phy_data = (u16)nes_read_indexed(nesdev,
NES_IDX_MAC_MDIO_CONTROL);
if ((phy_data == temp_phy_data) || (!(--u32temp)))
@@ -1675,6 +1691,14 @@ struct net_device *nes_netdev_init(struct nes_device *nesdev,
nes_debug(NES_DBG_INIT, "The Link is UP!!.\n");
nesvnic->linkup = 1;
}
+ } else if (nesdev->nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_PUMA_1G) {
+ nes_debug(NES_DBG_INIT, "mac_index=%d, logical_port=%d, u32temp=0x%04X, PCI_FUNC=%d\n",
+ nesdev->mac_index, nesvnic->logical_port, u32temp, PCI_FUNC(nesdev->pcidev->devfn));
+ if (((nesdev->mac_index < 2) && ((u32temp&0x01010000) == 0x01010000)) ||
+ ((nesdev->mac_index > 1) && ((u32temp&0x02020000) == 0x02020000))) {
+ nes_debug(NES_DBG_INIT, "The Link is UP!!.\n");
+ nesvnic->linkup = 1;
+ }
}
/* clear the MAC interrupt status, assumes direct logical to physical mapping */
u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (0x200 * nesdev->mac_index));
diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c
index c4ec6ac..cbd50fb 100644
--- a/drivers/infiniband/hw/nes/nes_utils.c
+++ b/drivers/infiniband/hw/nes/nes_utils.c
@@ -444,15 +444,13 @@ void nes_read_1G_phy_reg(struct nes_device *nesdev, u8 phy_reg, u8 phy_addr, u16
/**
* nes_write_10G_phy_reg
*/
-void nes_write_10G_phy_reg(struct nes_device *nesdev, u16 phy_reg,
- u8 phy_addr, u16 data)
+void nes_write_10G_phy_reg(struct nes_device *nesdev, u16 phy_addr, u8 dev_addr, u16 phy_reg,
+ u16 data)
{
- u32 dev_addr;
u32 port_addr;
u32 u32temp;
u32 counter;
- dev_addr = 1;
port_addr = phy_addr;
/* set address */
@@ -492,14 +490,12 @@ void nes_write_10G_phy_reg(struct nes_device *nesdev, u16 phy_reg,
* This routine only issues the read, the data must be read
* separately.
*/
-void nes_read_10G_phy_reg(struct nes_device *nesdev, u16 phy_reg, u8 phy_addr)
+void nes_read_10G_phy_reg(struct nes_device *nesdev, u8 phy_addr, u8 dev_addr, u16 phy_reg)
{
- u32 dev_addr;
u32 port_addr;
u32 u32temp;
u32 counter;
- dev_addr = 1;
port_addr = phy_addr;
/* set address */
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 1b3032f..ae12470 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -1043,10 +1043,10 @@ static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,
u8 sq_pbl_entries;
pbl_entries = nespbl->pbl_size >> 3;
- nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%p\n",
+ nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n",
nespbl->pbl_size, pbl_entries,
(void *)nespbl->pbl_vbase,
- (void *)nespbl->pbl_pbase);
+ (unsigned long) nespbl->pbl_pbase);
pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */
/* now lets set the sq_vbase as well as rq_vbase addrs we will assign */
/* the first pbl to be fro the rq_vbase... */
@@ -1074,9 +1074,9 @@ static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,
/* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */
/*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */
- nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%p rq_vbase=%p rq_pbase=%p\n",
- nesqp->hwqp.sq_vbase, (void *)nesqp->hwqp.sq_pbase,
- nesqp->hwqp.rq_vbase, (void *)nesqp->hwqp.rq_pbase);
+ nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n",
+ nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase,
+ nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase);
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
if (!nesadapter->free_256pbl) {
pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
@@ -1972,7 +1972,7 @@ static int nes_destroy_cq(struct ib_cq *ib_cq)
if (nescq->cq_mem_size)
pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size,
- (void *)nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase);
+ nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase);
kfree(nescq);
return ret;
@@ -2653,10 +2653,10 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
nespbl->pbl_vbase = (u64 *)pbl;
nespbl->user_base = start;
- nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%p,"
+ nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%lx,"
" pbl_vbase=%p user_base=0x%lx\n",
- nespbl->pbl_size, (void *)nespbl->pbl_pbase,
- (void*)nespbl->pbl_vbase, nespbl->user_base);
+ nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase,
+ (void *) nespbl->pbl_vbase, nespbl->user_base);
list_for_each_entry(chunk, ®ion->chunk_list, list) {
for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) {
@@ -3602,6 +3602,12 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
while (cqe_count < num_entries) {
if (le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) &
NES_CQE_VALID) {
+ /*
+ * Make sure we read CQ entry contents *after*
+ * we've checked the valid bit.
+ */
+ rmb();
+
cqe = nescq->hw_cq.cq_vbase[head];
nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
@@ -3900,9 +3906,6 @@ void nes_unregister_ofa_device(struct nes_ib_device *nesibdev)
struct nes_vnic *nesvnic = nesibdev->nesvnic;
int i;
- if (nesibdev == NULL)
- return;
-
for (i = 0; i < ARRAY_SIZE(nes_class_attributes); ++i) {
class_device_remove_file(&nesibdev->ibdev.class_dev, nes_class_attributes[i]);
}
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index ce3b4ca..16fc866 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -38,6 +38,7 @@
#include <net/icmp.h>
#include <linux/icmpv6.h>
#include <linux/delay.h>
+#include <linux/vmalloc.h>
int ipoib_max_conn_qp = 128;
@@ -233,7 +234,10 @@ static void ipoib_cm_free_rx_ring(struct net_device *dev,
dev_kfree_skb_any(rx_ring[i].skb);
}
- ipoib_vfree(&priv->cm.rx_vmap_srq_ring);
+ if (ipoib_cm_has_srq(dev))
+ ipoib_vfree(&priv->cm.rx_vmap_srq_ring);
+ else
+ vfree(rx_ring);
}
static void ipoib_cm_start_rx_drain(struct ipoib_dev_priv *priv)
@@ -360,9 +364,12 @@ static int ipoib_cm_nonsrq_init_rx(struct net_device *dev, struct ib_cm_id *cm_i
int ret;
int i;
- rx->rx_ring = kcalloc(ipoib_recvq_size, sizeof *rx->rx_ring, GFP_KERNEL);
- if (!rx->rx_ring)
+ rx->rx_ring = vmalloc(ipoib_recvq_size * sizeof *rx->rx_ring);
+ if (!rx->rx_ring){
+ printk(KERN_WARNING "ib: Allocation of rx_ring failed,%s\n",
+ "try using a lower value of recv_queue_size.");
return -ENOMEM;
+ }
spin_lock_irq(&priv->lock);
@@ -614,9 +621,14 @@ void ipoib_cm_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
if (wc->byte_len < SKB_TSHOLD) {
int dlen = wc->byte_len;
- small_skb = dev_alloc_skb(dlen);
+ small_skb = dev_alloc_skb(dlen + 12);
if (small_skb) {
+ skb_reserve(small_skb, 12);
+ ib_dma_sync_single_for_cpu(priv->ca, rx_ring[wr_id].mapping[0],
+ dlen, DMA_FROM_DEVICE);
skb_copy_from_linear_data(skb, small_skb->data, dlen);
+ ib_dma_sync_single_for_device(priv->ca, rx_ring[wr_id].mapping[0],
+ dlen, DMA_FROM_DEVICE);
skb_put(small_skb, dlen);
skb = small_skb;
goto copied;
@@ -734,8 +746,9 @@ void ipoib_cm_send(struct net_device *dev, struct sk_buff *skb, struct ipoib_cm_
dev->trans_start = jiffies;
++tx->tx_head;
- if (++priv->tx_outstanding == ipoib_sendq_size) {
- ipoib_dbg(priv, "TX ring 0x%x full, stopping kernel net queue\n",
+ if (++priv->tx_outstanding == ipoib_sendq_size - 1) {
+ ipoib_dbg(priv, "%s: TX ring 0x%x full,"
+ "stopping kernel net queue\n", __func__,
tx->qp->qp_num);
netif_stop_queue(dev);
}
@@ -1049,9 +1062,9 @@ static int ipoib_cm_modify_tx_init(struct net_device *dev,
struct ipoib_dev_priv *priv = netdev_priv(dev);
struct ib_qp_attr qp_attr;
int qp_attr_mask, ret;
- ret = ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &qp_attr.pkey_index);
+ ret = ib_find_pkey(priv->ca, priv->port, priv->pkey, &qp_attr.pkey_index);
if (ret) {
- ipoib_warn(priv, "pkey 0x%x not in cache: %d\n", priv->pkey, ret);
+ ipoib_warn(priv, "pkey 0x%x not found: %d\n", priv->pkey, ret);
return ret;
}
@@ -1424,6 +1437,11 @@ static ssize_t set_mode(struct class_device *d, const char *buf, size_t count)
priv->tx_wr.send_flags &= ~IB_SEND_IP_CSUM;
+ if (ipoib_cm_max_mtu(dev) > priv->mcast_mtu)
+ ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",
+ priv->mcast_mtu);
+ dev_set_mtu(dev, ipoib_cm_max_mtu(dev));
+
ipoib_flush_paths(dev);
return count;
}
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index cba3dfb..2f760fb 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -471,7 +471,11 @@ static void ipoib_ib_tx_timer_func(unsigned long dev_ptr)
ipoib_warn(priv, "failed to post zlen send\n");
else {
++priv->tx_head;
- ++priv->tx_outstanding;
+ if (++priv->tx_outstanding == ipoib_sendq_size - 1) {
+ ipoib_dbg(priv, "%s: TX ring full, "
+ "stopping kernel net queue\n", __func__);
+ netif_stop_queue(dev);
+ }
}
}
poll_tx(priv);
@@ -719,7 +723,7 @@ static void ipoib_pkey_dev_check_presence(struct net_device *dev)
struct ipoib_dev_priv *priv = netdev_priv(dev);
u16 pkey_index = 0;
- if (ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
+ if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
else
set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
@@ -966,13 +970,13 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int pkey_event)
clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
ipoib_ib_dev_down(dev, 0);
ipoib_ib_dev_stop(dev, 0);
- ipoib_pkey_dev_delay_open(dev);
- return;
+ if (ipoib_pkey_dev_delay_open(dev))
+ return;
}
- set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
/* restart QP only if P_Key index is changed */
- if (new_index == priv->pkey_index) {
+ if (test_and_set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags) &&
+ new_index == priv->pkey_index) {
ipoib_dbg(priv, "Not flushing - P_Key index not changed.\n");
return;
}
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index f30e854..e95e58d 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -854,6 +854,9 @@ static void ipoib_neigh_cleanup(struct neighbour *n)
unsigned long flags;
struct ipoib_ah *ah = NULL;
+ if (n->dev->type != ARPHRD_INFINIBAND)
+ return;
+
neigh = *to_ipoib_neigh(n);
if (neigh) {
priv = netdev_priv(neigh->dev);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 1bf45b0..b95e504 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -194,7 +194,13 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
/* Set the cached Q_Key before we attach if it's the broadcast group */
if (!memcmp(mcast->mcmember.mgid.raw, priv->dev->broadcast + 4,
sizeof (union ib_gid))) {
+ spin_lock_irq(&priv->lock);
+ if (!priv->broadcast) {
+ spin_unlock_irq(&priv->lock);
+ return -EAGAIN;
+ }
priv->qkey = be32_to_cpu(priv->broadcast->mcmember.qkey);
+ spin_unlock_irq(&priv->lock);
priv->tx_wr.wr.ud.remote_qkey = priv->qkey;
}
@@ -688,7 +694,7 @@ void ipoib_mcast_send(struct net_device *dev, void *mgid, struct sk_buff *skb)
*/
spin_lock(&priv->lock);
- if (!test_bit(IPOIB_MCAST_STARTED, &priv->flags) ||
+ if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags) ||
!priv->broadcast ||
!test_bit(IPOIB_MCAST_FLAG_ATTACHED, &priv->broadcast->flags)) {
++priv->stats.tx_dropped;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index d3bdbd3..e5ab3f9 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -173,10 +173,15 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
goto out_free_pd;
}
- size = ipoib_sendq_size + ipoib_recvq_size;
+ size = ipoib_recvq_size;
ret = ipoib_cm_dev_init(dev);
- if (!ret)
- size += ipoib_recvq_size + 1; /* 1 extra for rx_drain_qp */
+ if (!ret) {
+ size += ipoib_sendq_size;
+ if (ipoib_cm_has_srq(dev))
+ size += ipoib_recvq_size + 1; /* 1 extra for rx_drain_qp */
+ else
+ size += ipoib_recvq_size * ipoib_max_conn_qp;
+ }
priv->rcq = ib_create_cq(priv->ca, ipoib_ib_rx_completion, NULL, dev, size, 0);
if (IS_ERR(priv->rcq)) {
@@ -228,8 +233,9 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
priv->rx_wr_draft[i].sg_list = &priv->sglist_draft[i][0];
if (i < UD_POST_RCV_COUNT - 1)
priv->rx_wr_draft[i].next = &priv->rx_wr_draft[i + 1];
+ else
+ priv->rx_wr_draft[i].next = NULL;
}
- priv->rx_wr_draft[i].next = NULL;
if (ipoib_ud_need_sg(priv->max_ib_mtu)) {
for (i = 0; i < UD_POST_RCV_COUNT; ++i) {
diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c
index c85d412..c46fd60 100644
--- a/drivers/infiniband/ulp/sdp/sdp_main.c
+++ b/drivers/infiniband/ulp/sdp/sdp_main.c
@@ -2349,9 +2349,6 @@ static int __init sdp_proc_init(void)
p = proc_net_fops_create(sdp_seq_afinfo.name, S_IRUGO, sdp_seq_afinfo.seq_fops);
if (p)
p->data = &sdp_seq_afinfo;
- p = proc_net_fops_create(sdp_seq_afinfo.name, S_IRUGO, sdp_seq_afinfo.seq_fops);
- if (p)
- p->data = &sdp_seq_afinfo;
else
rc = -ENOMEM;
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 680cff5..332219f 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -81,6 +81,13 @@ module_param(mellanox_workarounds, int, 0444);
MODULE_PARM_DESC(mellanox_workarounds,
"Enable workarounds for Mellanox SRP target bugs if != 0");
+static int srp_dev_loss_tmo = 60;
+
+module_param(srp_dev_loss_tmo, int, 0444);
+MODULE_PARM_DESC(srp_dev_loss_tmo,
+ "Default number of seconds that srp transport should \
+ insulate the lost of a remote port (default is 60 secs");
+
static void srp_add_one(struct ib_device *device);
static void srp_remove_one(struct ib_device *device);
static void srp_completion(struct ib_cq *cq, void *target_ptr);
@@ -540,9 +547,10 @@ static void srp_remove_req(struct srp_target_port *target, struct srp_request *r
list_move_tail(&req->list, &target->free_reqs);
}
-static void srp_reset_req(struct srp_target_port *target, struct srp_request *req)
+static void srp_reset_req(struct srp_target_port *target,
+ struct srp_request *req, int status)
{
- req->scmnd->result = DID_RESET << 16;
+ req->scmnd->result = status << 16;
req->scmnd->scsi_done(req->scmnd);
srp_remove_req(target, req);
}
@@ -585,7 +593,7 @@ static int srp_reconnect_target(struct srp_target_port *target)
spin_lock_irq(target->scsi_host->host_lock);
list_for_each_entry_safe(req, tmp, &target->req_queue, list)
- srp_reset_req(target, req);
+ srp_reset_req(target, req, DID_RESET);
spin_unlock_irq(target->scsi_host->host_lock);
target->rx_head = 0;
@@ -931,13 +939,38 @@ static void srp_reconnect_work(struct work_struct *work)
static void srp_qp_in_err_timer(unsigned long data)
{
struct srp_target_port *target = (struct srp_target_port *)data;
+ struct srp_request *req, *tmp;
+
+ if (!target->qp_in_error || target->state != SRP_TARGET_LIVE)
+ goto out;
+
+ shost_printk(KERN_ERR, target->scsi_host,
+ PFX "srp_qp_in_err_timer called\n");
spin_lock_irq(target->scsi_host->host_lock);
+ list_for_each_entry_safe(req, tmp, &target->req_queue, list)
+ srp_reset_req(target, req, DID_NO_CONNECT);
+
INIT_WORK(&target->work, srp_reconnect_work);
schedule_work(&target->work);
spin_unlock_irq(target->scsi_host->host_lock);
+out:
+ if (target->qp_err_timer.function)
+ del_timer(&target->qp_err_timer);
+}
- del_timer(&target->qp_err_timer);
+static void srp_qp_err_add_timer(struct srp_target_port *target, int time)
+{
+ target->qp_in_error = 1;
+ if (!timer_pending(&target->qp_err_timer)) {
+ setup_timer(&target->qp_err_timer,
+ srp_qp_in_err_timer,
+ (unsigned long)target);
+ target->qp_err_timer.expires = time * HZ + jiffies;
+ add_timer(&target->qp_err_timer);
+
+ shost_printk(KERN_WARNING PFX, target->scsi_host, "add qp_in_err timer\n");
+ }
}
static void srp_completion(struct ib_cq *cq, void *target_ptr)
@@ -952,16 +985,10 @@ static void srp_completion(struct ib_cq *cq, void *target_ptr)
PFX "failed %s status %d\n",
wc.wr_id & SRP_OP_RECV ? "receive" : "send",
wc.status);
- if (!target->qp_in_error) {
- target->qp_in_error = 1;
- if (!timer_pending(&target->qp_err_timer)) {
- setup_timer(&target->qp_err_timer,
- srp_qp_in_err_timer,
- (unsigned long)target);
- target->qp_err_timer.expires = 10 * HZ + jiffies;
- add_timer(&target->qp_err_timer);
- }
- }
+ if (!target->qp_in_error &&
+ target->state == SRP_TARGET_LIVE)
+ srp_qp_err_add_timer(target,
+ srp_dev_loss_tmo - 30);
break;
}
@@ -1482,7 +1509,7 @@ static int srp_reset_device(struct scsi_cmnd *scmnd)
list_for_each_entry_safe(req, tmp, &target->req_queue, list)
if (req->scmnd->device == scmnd->device)
- srp_reset_req(target, req);
+ srp_reset_req(target, req, DID_RESET);
spin_unlock_irq(target->scsi_host->host_lock);
@@ -1492,9 +1519,21 @@ static int srp_reset_device(struct scsi_cmnd *scmnd)
static int srp_reset_host(struct scsi_cmnd *scmnd)
{
struct srp_target_port *target = host_to_target(scmnd->device->host);
+ struct srp_request *req, *tmp;
int ret = FAILED;
- shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
+ shost_printk(KERN_ERR, target->scsi_host,
+ PFX "SRP reset_host called state %d qp_err %d\n",
+ target->state, target->qp_in_error);
+
+ if ((timer_pending(&target->qp_err_timer) && target->qp_in_error) ||
+ target->state != SRP_TARGET_LIVE) {
+ spin_lock_irq(target->scsi_host->host_lock);
+ list_for_each_entry_safe(req, tmp, &target->req_queue, list)
+ srp_reset_req(target, req, DID_RESET);
+ spin_unlock_irq(target->scsi_host->host_lock);
+ return SUCCESS;
+ }
if (!srp_reconnect_target(target))
ret = SUCCESS;
@@ -1612,6 +1651,48 @@ static ssize_t show_local_ib_device(struct class_device *cdev, char *buf)
return sprintf(buf, "%s\n", target->srp_host->dev->dev->name);
}
+static ssize_t srp_target_oofabric(struct class_device *cdev,
+ const char *buf, size_t count)
+{
+ struct srp_target_port *target = host_to_target(class_to_shost(cdev));
+
+ shost_printk(KERN_DEBUG, target->scsi_host, PFX
+ "Get async_event out-of-fabric at state=%d qp_err=%d\n",
+ target->state, target->qp_in_error);
+
+ if (target->state != SRP_TARGET_LIVE)
+ return -EINVAL;
+
+ spin_lock_irq(target->scsi_host->host_lock);
+ if (!target->qp_in_error)
+ srp_qp_err_add_timer(target, srp_dev_loss_tmo);
+ spin_unlock_irq(target->scsi_host->host_lock);
+
+ return count;
+}
+
+static ssize_t srp_target_infabric(struct class_device *cdev,
+ const char *buf, size_t count)
+{
+ struct srp_target_port *target = host_to_target(class_to_shost(cdev));
+
+ shost_printk(KERN_DEBUG, target->scsi_host, PFX
+ "Get async_event in-fabric at state=%d qp_err=%d\n",
+ target->state, target->qp_in_error);
+
+ spin_lock_irq(target->scsi_host->host_lock);
+ if (timer_pending(&target->qp_err_timer)
+ && target->qp_in_error) {
+ shost_printk(KERN_WARNING PFX, target->scsi_host, "delete qp_in_err timer\n");
+ del_timer(&target->qp_err_timer);
+ INIT_WORK(&target->work, srp_reconnect_work);
+ schedule_work(&target->work);
+ }
+ spin_unlock_irq(target->scsi_host->host_lock);
+
+ return count;
+}
+
static CLASS_DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
static CLASS_DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
static CLASS_DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
@@ -1621,6 +1702,8 @@ static CLASS_DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
static CLASS_DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
static CLASS_DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
static CLASS_DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
+static CLASS_DEVICE_ATTR(target_oofabric, S_IWUSR, NULL, srp_target_oofabric);
+static CLASS_DEVICE_ATTR(target_infabric, S_IWUSR, NULL, srp_target_infabric);
static struct class_device_attribute *srp_host_attrs[] = {
&class_device_attr_id_ext,
@@ -1632,6 +1715,8 @@ static struct class_device_attribute *srp_host_attrs[] = {
&class_device_attr_zero_req_lim,
&class_device_attr_local_ib_port,
&class_device_attr_local_ib_device,
+ &class_device_attr_target_oofabric,
+ &class_device_attr_target_infabric,
NULL
};
@@ -2020,6 +2105,71 @@ free_host:
return NULL;
}
+static void srp_event_handler(struct ib_event_handler *handler,
+ struct ib_event *event)
+{
+ struct srp_device *srp_dev =
+ ib_get_client_data(event->device, &srp_client);
+ struct srp_host *host, *tmp_host;
+ struct srp_target_port *target, *tmp_target;
+
+ if (!srp_dev || srp_dev->dev != event->device)
+ return;
+
+ printk(KERN_DEBUG PFX "ASYNC event=%d on device=%s\n",
+ event->event, srp_dev->dev->name);
+
+ switch (event->event) {
+ case IB_EVENT_PORT_ERR:
+ list_for_each_entry_safe(host, tmp_host,
+ &srp_dev->dev_list, list) {
+ if (event->element.port_num == host->port) {
+ spin_lock(&host->target_lock);
+ list_for_each_entry_safe(target, tmp_target,
+ &host->target_list, list) {
+ spin_lock_irq(target->scsi_host->host_lock);
+ if (!target->qp_in_error &&
+ target->state == SRP_TARGET_LIVE)
+ srp_qp_err_add_timer(target,
+ srp_dev_loss_tmo);
+ spin_unlock_irq(target->scsi_host->host_lock);
+ }
+ spin_unlock(&host->target_lock);
+ }
+ }
+ break;
+ case IB_EVENT_PORT_ACTIVE:
+ case IB_EVENT_LID_CHANGE:
+ case IB_EVENT_PKEY_CHANGE:
+ case IB_EVENT_SM_CHANGE:
+ list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list,
+ list) {
+ if (event->element.port_num == host->port) {
+ spin_lock(&host->target_lock);
+ list_for_each_entry_safe(target, tmp_target,
+ &host->target_list, list) {
+ spin_lock_irq(target->scsi_host->host_lock);
+ if (timer_pending(&target->qp_err_timer)
+ && target->qp_in_error) {
+ shost_printk(KERN_WARNING PFX,
+ target->scsi_host,
+ "delete qp_in_err timer\n");
+ del_timer(&target->qp_err_timer);
+ INIT_WORK(&target->work, srp_reconnect_work);
+ schedule_work(&target->work);
+ }
+ spin_unlock_irq(target->scsi_host->host_lock);
+ }
+ spin_unlock(&host->target_lock);
+ }
+ }
+ break;
+ default:
+ break;
+ }
+
+}
+
static void srp_add_one(struct ib_device *device)
{
struct srp_device *srp_dev;
@@ -2065,6 +2215,11 @@ static void srp_add_one(struct ib_device *device)
if (IS_ERR(srp_dev->mr))
goto err_pd;
+ INIT_IB_EVENT_HANDLER(&srp_dev->event_handler, srp_dev->dev,
+ srp_event_handler);
+ if (ib_register_event_handler(&srp_dev->event_handler))
+ goto err_pd;
+
memset(&fmr_param, 0, sizeof fmr_param);
fmr_param.pool_size = SRP_FMR_POOL_SIZE;
fmr_param.dirty_watermark = SRP_FMR_DIRTY_SIZE;
@@ -2116,6 +2271,8 @@ static void srp_remove_one(struct ib_device *device)
srp_dev = ib_get_client_data(device, &srp_client);
+ ib_unregister_event_handler(&srp_dev->event_handler);
+
list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
class_device_unregister(&host->class_dev);
/*
@@ -2172,6 +2329,9 @@ static int __init srp_init_module(void)
sizeof (struct srp_indirect_buf) +
srp_sg_tablesize * 16);
+ if (srp_dev_loss_tmo < 60)
+ srp_dev_loss_tmo = 60;
+
ret = class_register(&srp_class);
if (ret) {
printk(KERN_ERR PFX "couldn't register class infiniband_srp\n");
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index 631d8bd..93607a4 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -90,6 +90,7 @@ struct srp_device {
struct ib_device *dev;
struct ib_pd *pd;
struct ib_mr *mr;
+ struct ib_event_handler event_handler;
struct ib_fmr_pool *fmr_pool;
int fmr_page_shift;
int fmr_page_size;
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index cb8de45..d8c7515 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -46,9 +46,9 @@ enum {
extern void __buggy_use_of_MLX4_GET(void);
extern void __buggy_use_of_MLX4_PUT(void);
-static int mlx4_core_enable_qos = 1;
+static int mlx4_core_enable_qos = 0;
module_param_named(enable_qos, mlx4_core_enable_qos, int, 0444);
-MODULE_PARM_DESC(enable_qos, "Enable Quality of Service support in the HCA if > 0, (default 1)");
+MODULE_PARM_DESC(enable_qos, "Enable Quality of Service support in the HCA if > 0, (default 0)");
#define MLX4_GET(dest, source, offset) \
do { \
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index ea1ebdc..bda8839 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -791,8 +791,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
}
/*
- * Check for BARs. We expect 0: 1MB, 2: 8MB, 4: DDR (may not
- * be present)
+ * Check for BARs. We expect 0: 1MB
*/
if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
pci_resource_len(pdev, 0) != 1 << 20) {
diff --git a/drivers/net/mlx4/mcg.c b/drivers/net/mlx4/mcg.c
index 6da6a43..7aa375b 100644
--- a/drivers/net/mlx4/mcg.c
+++ b/drivers/net/mlx4/mcg.c
@@ -190,10 +190,6 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
}
index += dev->caps.num_mgms;
- err = mlx4_READ_MCG(dev, index, mailbox);
- if (err)
- goto out;
-
memset(mgm, 0, sizeof *mgm);
memcpy(mgm->gid, gid, 16);
}
@@ -302,12 +298,10 @@ int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
mgm->qp[loc] = mgm->qp[i - 1];
mgm->qp[i - 1] = 0;
- err = mlx4_WRITE_MCG(dev, index, mailbox);
- if (err)
- goto out;
-
- if (i != 1)
+ if (i != 1) {
+ err = mlx4_WRITE_MCG(dev, index, mailbox);
goto out;
+ }
if (prev == -1) {
/* Remove entry from MGM */
diff --git a/drivers/net/mlx4/mr.c b/drivers/net/mlx4/mr.c
index 730b3dc..7d63d7c 100644
--- a/drivers/net/mlx4/mr.c
+++ b/drivers/net/mlx4/mr.c
@@ -551,7 +551,7 @@ int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
u64 mtt_seg;
int err = -ENOMEM;
- if (page_shift < 12 || page_shift >= 32)
+ if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32)
return -EINVAL;
/* All MTTs must fit in the same page */
@@ -607,15 +607,9 @@ EXPORT_SYMBOL_GPL(mlx4_fmr_enable);
void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
u32 *lkey, u32 *rkey)
{
- u32 key;
-
if (!fmr->maps)
return;
- key = key_to_hw_index(fmr->mr.key);
- key &= dev->caps.num_mpts - 1;
- *lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
-
fmr->maps = 0;
*(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
diff --git a/drivers/net/mlx4/qp.c b/drivers/net/mlx4/qp.c
index 1692a4b..f5f446c 100644
--- a/drivers/net/mlx4/qp.c
+++ b/drivers/net/mlx4/qp.c
@@ -55,7 +55,7 @@ void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type)
spin_unlock(&qp_table->lock);
if (!qp) {
- mlx4_dbg(dev, "Async event %d for bogus QP %08x\n",
+ mlx4_warn(dev, "Async event %d for bogus QP %08x\n",
event_type, qpn);
return;
}
distrib
>
Scientific%20Linux
>
5x
>
x86_64
>
by-pkgid
>
89877e42827f16fa5f86b1df0c2860b1
>
files
>
1714
